level=info ts=2026-02-16T11:30:57.855752447Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T11:30:57.855790436Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-02-16T11:30:57.856974402Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T11:30:57.858917045Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T11:30:57.859024019Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.69 inf=eth0
level=info ts=2026-02-16T11:30:57.859572442Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-querier-77cddbcfd9-czznf-71e050cb
ts=2026-02-16T11:30:57.85966956Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T11:30:57.859684712Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.69 advertisePort=7946
level=debug ts=2026-02-16T11:30:57.859711532Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.69 advertisePort=7946
level=info ts=2026-02-16T11:30:57.859850412Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-02-16T11:30:57.863349837Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T11:30:57.863378757Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
ts=2026-02-16T11:30:57.86531374Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T11:30:57.865333044Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T11:30:57.865359815Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.511553ms
level=error ts=2026-02-16T11:30:57.865372377Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T11:30:57.865404006Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T11:30:57.867777049Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:30:57.867792458Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:30:57.868351322Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:30:57.868375064Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T11:30:57.868743608Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=debug ts=2026-02-16T11:30:57.869029982Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=info ts=2026-02-16T11:30:57.869065682Z caller=worker.go:138 component=querier msg="Starting querier worker connected to query-scheduler" scheduler=loki-query-scheduler.product-kubearchive-logging.svc.cluster.local:9095
level=debug ts=2026-02-16T11:30:57.887269619Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-02-16T11:30:57.887271011Z caller=module_service.go:82 msg=starting module=cache-generation-loader
level=info ts=2026-02-16T11:30:57.887288345Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-02-16T11:30:57.887306396Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=analytics
level=debug ts=2026-02-16T11:30:57.887307065Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=ring
level=info ts=2026-02-16T11:30:57.887312926Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-02-16T11:30:57.88732116Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-02-16T11:30:57.887323918Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-02-16T11:30:57.887332091Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ingester-querier
level=info ts=2026-02-16T11:30:57.887349572Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T11:30:57.887362845Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-02-16T11:30:57.887379771Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-02-16T11:30:57.887475782Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=cache-generation-loader
level=debug ts=2026-02-16T11:30:57.887500571Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ingester-querier
level=info ts=2026-02-16T11:30:57.887554955Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-02-16T11:30:57.887614105Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-02-16T11:30:57.887648697Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=runtime-config
level=debug ts=2026-02-16T11:30:57.887655281Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=server
level=debug ts=2026-02-16T11:30:57.88765874Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=memberlist-kv
level=info ts=2026-02-16T11:30:57.887663449Z caller=module_service.go:82 msg=starting module=ingester-querier
level=debug ts=2026-02-16T11:30:57.887674193Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=memberlist-kv
level=debug ts=2026-02-16T11:30:57.887679438Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ring
level=debug ts=2026-02-16T11:30:57.887684336Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=runtime-config
level=debug ts=2026-02-16T11:30:57.887700596Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=server
level=debug ts=2026-02-16T11:30:57.887703909Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=store
level=debug ts=2026-02-16T11:30:57.887708803Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-02-16T11:30:57.887718949Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ring
level=debug ts=2026-02-16T11:30:57.887723918Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-02-16T11:30:57.887728558Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-02-16T11:30:57.887733174Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-02-16T11:30:57.887750073Z caller=module_service.go:82 msg=starting module=querier
level=info ts=2026-02-16T11:30:57.887792801Z caller=loki.go:599 msg="Loki started" startup_time=91.158602ms
level=warn ts=2026-02-16T11:30:57.891178584Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-02-16T11:30:59.532754521Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:30:59.532780143Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T11:31:00.893106016Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-02-16T11:31:02.714253111Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:02.714279491Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:31:02.866642917Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:02.86667222Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-02-16T11:31:03.895478196Z caller=worker.go:235 component=querier msg="adding connection" addr=10.129.0.71:9095
ts=2026-02-16T11:31:06.866292692Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:06.866317429Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:31:07.867512563Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:07.867540508Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=error ts=2026-02-16T11:31:12.86714574Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:12.867172604Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T11:31:14.241100601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:39252"
level=debug ts=2026-02-16T11:31:16.027718186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:31:17.866644474Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:17.866668867Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T11:31:18.454519832Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.59:7946: dial tcp 10.130.0.59:7946: connect: connection refused"
ts=2026-02-16T11:31:18.455104779Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.71:7946: dial tcp 10.129.0.71:7946: connect: connection refused"
ts=2026-02-16T11:31:18.455213099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.69:7946"
ts=2026-02-16T11:31:18.455237846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:43766"
level=info ts=2026-02-16T11:31:18.455688247Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=20.590277516s
ts=2026-02-16T11:31:20.815655779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:56840"
level=error ts=2026-02-16T11:31:22.866626159Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:22.866650336Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T11:31:23.142388393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:43984"
level=debug ts=2026-02-16T11:31:35.141639548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:31:45.751364878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:31:57.864711162Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=debug ts=2026-02-16T11:31:58.326555897Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:32:15.75390823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:32:17.075135552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T11:32:42.048130397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:32:42.050018184Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:32:45.756126213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:32:57.864781431Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:32:59.02898898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:33210"
ts=2026-02-16T11:33:00.133590705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47224"
ts=2026-02-16T11:33:15.371546652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:56922"
ts=2026-02-16T11:33:15.758617017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:33:22.01809666Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:33:30.13544765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:50002"
ts=2026-02-16T11:33:45.761143097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:33:57.865467446Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:34:15.764165257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T11:34:30.138090271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:50302"
level=debug ts=2026-02-16T11:34:35.274646009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:34:45.767186153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:34:57.864792251Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:35:00.139689163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:50494"
ts=2026-02-16T11:35:15.3810079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:36048"
ts=2026-02-16T11:35:15.76917785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:35:24.573677851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:35:29.044500515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:39600"
ts=2026-02-16T11:35:45.771467824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:35:57.865019989Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:36:15.773478112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:36:22.85496604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:36:29.050205415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:41190"
ts=2026-02-16T11:36:45.776050906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:36:57.86442639Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:37:15.778194121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T11:37:29.056145971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:52392"
ts=2026-02-16T11:37:30.150134071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:50638"
level=debug ts=2026-02-16T11:37:40.345520461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:37:40.347127328Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:37:45.780477871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:37:57.864954299Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:38:15.396085843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:56732"
ts=2026-02-16T11:38:15.781972236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:38:30.153633279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41612"
level=debug ts=2026-02-16T11:38:42.450803908Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:38:45.783932847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:38:57.86500068Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:38:59.064940725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:40576"
ts=2026-02-16T11:39:15.78566088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:39:30.15904513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:33488"
level=debug ts=2026-02-16T11:39:37.462104076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:39:45.787843333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:39:57.864875177Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:40:15.789992298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:40:20.859776215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:40:29.073326425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:47292"
ts=2026-02-16T11:40:30.163019967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:53656"
ts=2026-02-16T11:40:45.403290032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:37144"
ts=2026-02-16T11:40:45.79273847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:40:57.86484476Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:41:00.165036046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:56312"
level=debug ts=2026-02-16T11:41:01.285279118Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:41:15.795721626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T11:41:30.167488289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42212"
ts=2026-02-16T11:41:45.798136117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:41:50.577277517Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:41:50.578788146Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T11:41:57.865112005Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:42:15.409413607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:49876"
ts=2026-02-16T11:42:15.80053027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:42:38.196116762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:42:45.802894681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:42:57.864450764Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:42:59.086939437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:46518"
ts=2026-02-16T11:43:00.174690831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:43148"
ts=2026-02-16T11:43:15.414273866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:52946"
ts=2026-02-16T11:43:15.805295076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:43:23.435626319Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:43:30.176487781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:51018"
ts=2026-02-16T11:43:45.808016319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:43:57.865551451Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=debug ts=2026-02-16T11:44:04.905016245Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:44:15.81077659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T11:44:45.813526611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:44:57.86483693Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:44:59.098929361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:36212"
level=debug ts=2026-02-16T11:45:06.560287756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:45:15.422343034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:53590"
ts=2026-02-16T11:45:15.815822597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T11:45:29.101872764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:54314"
ts=2026-02-16T11:45:30.185295172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:38454"
ts=2026-02-16T11:45:45.818428119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:45:56.438463313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:45:56.440061315Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T11:45:57.865199652Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:46:15.820783812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:46:29.108017154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:51388"
ts=2026-02-16T11:46:45.430685733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:36774"
ts=2026-02-16T11:46:45.822721239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:46:50.699686963Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T11:46:57.865192166Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:47:00.192847052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:36936"
ts=2026-02-16T11:47:15.8253927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:47:30.194634118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:51960"
ts=2026-02-16T11:47:45.435834289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:51740"
ts=2026-02-16T11:47:45.827389674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:47:46.097379057Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T11:47:57.865115159Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:48:15.829858634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:48:28.444020425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:48:45.832377348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:48:57.864298572Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:49:15.441367686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:45480"
ts=2026-02-16T11:49:15.83441394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:49:23.578721661Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:49:30.20228544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:36406"
ts=2026-02-16T11:49:45.443562457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:43900"
ts=2026-02-16T11:49:45.836741256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:49:57.865297329Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=debug ts=2026-02-16T11:50:02.23729757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:50:02.238848884Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:50:15.838644244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
ts=2026-02-16T11:50:45.841193175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:50:57.865119835Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:50:59.143255775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:41856"
ts=2026-02-16T11:51:00.210053803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:53646"
level=debug ts=2026-02-16T11:51:02.316765748Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:51:15.844178588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:51:38.766428357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:51:45.449279281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:57074"
ts=2026-02-16T11:51:45.847319175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:51:57.86511241Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:52:15.84964523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:52:24.521632866Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:52:29.152700213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:54590"
ts=2026-02-16T11:52:45.85216901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:52:57.864749649Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:52:59.155787856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:56790"
ts=2026-02-16T11:53:15.854964813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:53:17.912948596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:53:30.22161609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:37882"
ts=2026-02-16T11:53:45.857444583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:53:54.615710568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:53:54.617226979Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T11:53:57.86490715Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:54:15.45678471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:41626"
ts=2026-02-16T11:54:15.859369234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
ts=2026-02-16T11:54:30.226703869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35218"
level=debug ts=2026-02-16T11:54:30.642414329Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:54:45.861660662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:54:57.865247823Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:54:59.167552078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:52104"
ts=2026-02-16T11:55:15.863650275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:55:18.801993013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:55:30.231382904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:40858"
ts=2026-02-16T11:55:45.463001895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:42734"
ts=2026-02-16T11:55:45.86553847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:55:57.864641066Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:56:00.233653723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:36186"
level=debug ts=2026-02-16T11:56:06.888722208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:56:15.465233113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:48928"
ts=2026-02-16T11:56:15.867691739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:56:29.176360224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:52986"
ts=2026-02-16T11:56:45.870708192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:56:57.865247933Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=debug ts=2026-02-16T11:57:03.888761816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:57:15.87369104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
ts=2026-02-16T11:57:29.18138576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:59786"
level=debug ts=2026-02-16T11:57:40.113771687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:57:40.115702714Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:57:45.471340587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:56270"
ts=2026-02-16T11:57:45.876066211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:57:57.865327752Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T11:57:59.183921383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:47000"
ts=2026-02-16T11:58:15.879016441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:58:17.728437491Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:58:30.245261816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:55680"
ts=2026-02-16T11:58:45.881123686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:58:57.865367399Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=debug ts=2026-02-16T11:59:02.533871714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:59:15.8835217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T11:59:29.193205745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:37618"
ts=2026-02-16T11:59:30.249233924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:48500"
ts=2026-02-16T11:59:45.886962332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:59:57.86470754Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:00:00.251860646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:44304"
level=debug ts=2026-02-16T12:00:04.258022757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:00:15.890034235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:00:45.400301541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:00:45.892484409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:00:57.864290318Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:01:15.895445624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T12:01:37.768206425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:01:37.769700378Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:01:45.484940393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:59034"
ts=2026-02-16T12:01:45.898141032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T12:01:57.864956865Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:01:59.208114373Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:38384"
ts=2026-02-16T12:02:15.900518983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T12:02:30.869720082Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:02:45.902686962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:02:57.864339883Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=debug ts=2026-02-16T12:03:07.131312121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:03:15.904276275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:03:29.21703825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:36900"
ts=2026-02-16T12:03:45.906764632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T12:03:57.864864996Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:03:59.222388592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:47356"
level=debug ts=2026-02-16T12:04:00.302483357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:04:15.908971063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:04:29.225181173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:59494"
ts=2026-02-16T12:04:45.911450987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:04:49.7002258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:04:57.864837966Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:05:00.274614355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:52066"
ts=2026-02-16T12:05:15.502377459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:55468"
ts=2026-02-16T12:05:15.913704174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
ts=2026-02-16T12:05:45.91581901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T12:05:46.412630169Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:05:46.414965698Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T12:05:57.865099852Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:06:00.279198059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47340"
ts=2026-02-16T12:06:15.918698372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T12:06:30.280903204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:43054"
level=debug ts=2026-02-16T12:06:36.660490067Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:06:45.508228083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:44326"
ts=2026-02-16T12:06:45.921077506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:06:57.864385166Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=debug ts=2026-02-16T12:07:11.953743394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:07:15.923717046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:07:45.930556258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T12:07:57.865302094Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:08:00.286348312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:49950"
level=debug ts=2026-02-16T12:08:05.484508598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:08:15.936188779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
ts=2026-02-16T12:08:30.288668656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:37088"
ts=2026-02-16T12:08:45.51652952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:57682"
ts=2026-02-16T12:08:45.938338187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T12:08:48.321456766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:08:57.865267894Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:09:15.941340839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T12:09:41.770693077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:09:41.772368407Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:09:45.943651229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T12:09:57.865299622Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:10:15.946105495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T12:10:23.704356878Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:10:29.262684722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:35172"
ts=2026-02-16T12:10:45.949069094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T12:10:57.864625076Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:10:59.265408256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:33460"
ts=2026-02-16T12:11:15.527573474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:37912"
ts=2026-02-16T12:11:15.952318082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T12:11:16.065445251Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:11:29.268421036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:47976"
ts=2026-02-16T12:11:45.954405723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T12:11:57.864558562Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:11:59.271473171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:50810"
ts=2026-02-16T12:12:00.304806773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60124"
level=debug ts=2026-02-16T12:12:12.209291479Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:12:15.956868234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
ts=2026-02-16T12:12:29.275107252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:56726"
ts=2026-02-16T12:12:30.306783969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:33292"
ts=2026-02-16T12:12:45.959123565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T12:12:57.864348131Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:12:59.278255112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:40758"
ts=2026-02-16T12:13:00.308644778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:40700"
level=debug ts=2026-02-16T12:13:12.510229737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:13:15.961953853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
ts=2026-02-16T12:13:45.964334461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T12:13:57.865318803Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=debug ts=2026-02-16T12:14:13.776649894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:14:13.778166635Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:14:15.966779195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
ts=2026-02-16T12:14:30.311864243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35774"
ts=2026-02-16T12:14:45.968919434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:14:56.983878805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:14:57.864222925Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:15:00.314043881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42274"
ts=2026-02-16T12:15:15.971143665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
ts=2026-02-16T12:15:30.315731216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:59592"
level=debug ts=2026-02-16T12:15:43.211989485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:15:45.54652862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:45108"
ts=2026-02-16T12:15:45.9737204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:15:57.864766988Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:15:59.295740102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:54640"
ts=2026-02-16T12:16:00.317438013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47052"
ts=2026-02-16T12:16:15.97562681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
ts=2026-02-16T12:16:30.319492293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:34042"
level=debug ts=2026-02-16T12:16:43.085168529Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:16:45.549861297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:39280"
ts=2026-02-16T12:16:45.977646499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:16:57.865235423Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:17:15.551851163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:53744"
ts=2026-02-16T12:17:15.980778288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T12:17:19.327828777Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:17:45.983447976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T12:17:57.864751462Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:17:59.308108339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:33116"
ts=2026-02-16T12:18:00.325977946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:38196"
level=debug ts=2026-02-16T12:18:07.189857866Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:18:07.191538604Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:18:15.554942758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:39132"
ts=2026-02-16T12:18:15.985227697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
ts=2026-02-16T12:18:30.327771994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35808"
ts=2026-02-16T12:18:45.556895825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:38672"
ts=2026-02-16T12:18:45.987091096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T12:18:47.251544405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:18:57.864741364Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:19:15.558914204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:60558"
ts=2026-02-16T12:19:15.989354938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:19:30.330840324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:36784"
level=debug ts=2026-02-16T12:19:33.797010104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:19:45.991014806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:19:57.865369804Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:20:00.333003219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46684"
ts=2026-02-16T12:20:15.564240633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:50738"
ts=2026-02-16T12:20:15.992844288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
ts=2026-02-16T12:20:29.323121204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:39012"
level=debug ts=2026-02-16T12:20:29.515372952Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:20:45.566574278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:53478"
ts=2026-02-16T12:20:45.994352132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T12:20:57.864649052Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:20:59.325954286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:37524"
ts=2026-02-16T12:21:00.336383239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46282"
ts=2026-02-16T12:21:15.996697072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:21:24.598530904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:21:30.338792211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:33450"
ts=2026-02-16T12:21:45.999070847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:21:57.865202817Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=debug ts=2026-02-16T12:22:10.627678179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:22:10.629218935Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:22:15.573423762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:33414"
ts=2026-02-16T12:22:16.001375531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
ts=2026-02-16T12:22:45.576069096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:52176"
ts=2026-02-16T12:22:46.0042882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T12:22:57.099806901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:22:57.864527754Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
ts=2026-02-16T12:23:00.34602902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35772"
ts=2026-02-16T12:23:16.006645541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:23:29.340611479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:56120"
level=debug ts=2026-02-16T12:23:42.383884924Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:23:46.009509033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T12:23:57.865151631Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1