level=info ts=2026-02-16T05:56:26.376083521Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T05:56:26.376109807Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-02-16T05:56:26.378211854Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T05:56:26.378548955Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T05:56:26.378640931Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.57 inf=eth0
level=info ts=2026-02-16T05:56:26.379084635Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-querier-77cddbcfd9-2sdsw-273d962c
ts=2026-02-16T05:56:26.37915062Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T05:56:26.379162406Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.57 advertisePort=7946
level=debug ts=2026-02-16T05:56:26.379201584Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.57 advertisePort=7946
level=info ts=2026-02-16T05:56:26.379326178Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-02-16T05:56:26.382512804Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T05:56:26.382529848Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T05:56:26.382545398Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=3.220603ms
level=error ts=2026-02-16T05:56:26.38255249Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T05:56:26.382563184Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T05:56:26.384532446Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T05:56:26.384544145Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T05:56:26.385391921Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T05:56:26.385412911Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=error ts=2026-02-16T05:56:26.395773139Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T05:56:26.395791798Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T05:56:26.39608256Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=debug ts=2026-02-16T05:56:26.396320973Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=info ts=2026-02-16T05:56:26.396350186Z caller=worker.go:138 component=querier msg="Starting querier worker connected to query-scheduler" scheduler=loki-query-scheduler.product-kubearchive-logging.svc.cluster.local:9095
level=debug ts=2026-02-16T05:56:26.411628078Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ingester-querier
level=info ts=2026-02-16T05:56:26.41163096Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-02-16T05:56:26.411653502Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-02-16T05:56:26.411644692Z caller=module_service.go:82 msg=starting module=cache-generation-loader
level=debug ts=2026-02-16T05:56:26.411676697Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-02-16T05:56:26.411677412Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-02-16T05:56:26.411707303Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=store
level=info ts=2026-02-16T05:56:26.411706867Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-02-16T05:56:26.411709948Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T05:56:26.411919416Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=debug ts=2026-02-16T05:56:26.411931047Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-02-16T05:56:26.411934492Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-02-16T05:56:26.41196819Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-02-16T05:56:26.41256402Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=memberlist-kv
level=debug ts=2026-02-16T05:56:26.412580911Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=ring
level=debug ts=2026-02-16T05:56:26.412585087Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=runtime-config
level=debug ts=2026-02-16T05:56:26.412588269Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=server
level=info ts=2026-02-16T05:56:26.412591502Z caller=module_service.go:82 msg=starting module=ingester-querier
level=debug ts=2026-02-16T05:56:26.412607905Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-02-16T05:56:26.412612027Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ring
level=debug ts=2026-02-16T05:56:26.412614702Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-02-16T05:56:26.412617258Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-02-16T05:56:26.412619733Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-02-16T05:56:26.412630401Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=analytics
level=debug ts=2026-02-16T05:56:26.412633694Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=cache-generation-loader
level=debug ts=2026-02-16T05:56:26.412636505Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ingester-querier
level=debug ts=2026-02-16T05:56:26.412639218Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=memberlist-kv
level=debug ts=2026-02-16T05:56:26.412641824Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ring
level=debug ts=2026-02-16T05:56:26.412644861Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=runtime-config
level=debug ts=2026-02-16T05:56:26.412647373Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=server
level=info ts=2026-02-16T05:56:26.412649723Z caller=module_service.go:82 msg=starting module=querier
level=info ts=2026-02-16T05:56:26.412708718Z caller=loki.go:599 msg="Loki started" startup_time=113.089497ms
level=warn ts=2026-02-16T05:56:26.41718781Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-02-16T05:56:28.221244398Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T05:56:28.221278314Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T05:56:29.419759771Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T05:56:31.392563609Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T05:56:31.392599916Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T05:56:31.991458994Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T05:56:31.991487949Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T05:56:32.422319662Z caller=worker.go:235 component=querier msg="adding connection" addr=10.128.0.58:9095
level=error ts=2026-02-16T05:56:36.393411677Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T05:56:36.393441174Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T05:56:38.404821103Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T05:56:38.40484659Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T05:56:41.392741422Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T05:56:41.392768796Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T05:56:42.2639751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T05:56:42.953223224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:51498"
ts=2026-02-16T05:56:43.048492065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:53650"
ts=2026-02-16T05:56:45.559215329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:47324"
ts=2026-02-16T05:56:51.674575036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.57:7946"
ts=2026-02-16T05:56:51.674679183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:57084"
ts=2026-02-16T05:56:51.675451803Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.58:7946: dial tcp 10.128.0.58:7946: connect: connection refused"
ts=2026-02-16T05:56:51.676550252Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.59:7946: dial tcp 10.128.0.59:7946: connect: connection refused"
ts=2026-02-16T05:56:51.676758745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.60:7946"
level=info ts=2026-02-16T05:56:51.677290652Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=25.294722696s
ts=2026-02-16T05:57:01.120479301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T05:57:01.761093574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T05:57:05.703188238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:57018"
ts=2026-02-16T05:57:08.008979862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:37910"
level=debug ts=2026-02-16T05:57:26.386770624Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=debug ts=2026-02-16T05:57:26.395492848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T05:57:31.123916977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T05:57:46.925477083Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T05:58:01.126786183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T05:58:08.012376057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:47254"
level=debug ts=2026-02-16T05:58:12.52164692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T05:58:12.523808746Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T05:58:26.386615563Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T05:58:31.128962535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T05:58:58.006945437Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T05:59:01.130821055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T05:59:26.387084665Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T05:59:31.132205279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T05:59:36.119134376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:38666"
ts=2026-02-16T05:59:38.018345204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:58658"
ts=2026-02-16T06:00:01.13388804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:00:07.366223195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T06:00:26.387333007Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:00:31.13600646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:01:01.137337004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:01:05.714111502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:34632"
ts=2026-02-16T06:01:06.124499797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:60106"
level=debug ts=2026-02-16T06:01:18.825594644Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T06:01:26.387084824Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:01:31.138808727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:01:35.716483352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:44326"
ts=2026-02-16T06:02:01.141044748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:02:08.027640722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:32942"
level=debug ts=2026-02-16T06:02:13.780398519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T06:02:26.387199408Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:02:31.142899422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:02:35.720482579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:38306"
ts=2026-02-16T06:02:36.129828401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:48894"
ts=2026-02-16T06:02:38.029194055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:53716"
ts=2026-02-16T06:03:01.144838639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:03:15.727404342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T06:03:15.728939239Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T06:03:26.387053106Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:03:31.146669017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:03:38.033385756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:54642"
ts=2026-02-16T06:04:01.148873173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:04:02.720991305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:04:05.724930816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:57990"
ts=2026-02-16T06:04:06.135142342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:51016"
ts=2026-02-16T06:04:08.034425223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:55554"
level=debug ts=2026-02-16T06:04:26.386559194Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:04:31.15124837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:04:54.131376134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:05:01.153388749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:05:05.729041348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:47678"
ts=2026-02-16T06:05:06.139588071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:53658"
ts=2026-02-16T06:05:08.038140302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:50936"
level=debug ts=2026-02-16T06:05:26.38711073Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:05:31.155498311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:05:38.039462713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:57628"
level=debug ts=2026-02-16T06:05:39.846511844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:06:01.156525139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:06:05.733195548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:48798"
ts=2026-02-16T06:06:08.040990811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:50544"
level=debug ts=2026-02-16T06:06:24.425445199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:06:26.387672946Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:06:31.157764597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:06:36.143298363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:36368"
ts=2026-02-16T06:07:01.159292257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:07:05.737051422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:47382"
ts=2026-02-16T06:07:06.145448753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:51950"
ts=2026-02-16T06:07:08.044254349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:58040"
level=debug ts=2026-02-16T06:07:13.372394493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:07:13.374026097Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T06:07:26.387437434Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:07:31.160941549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:07:38.046014817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:56614"
ts=2026-02-16T06:08:01.162849032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:08:05.960588195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:08:06.148437385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:35174"
level=debug ts=2026-02-16T06:08:26.387040409Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:08:31.165295188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:08:35.742723301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:37624"
ts=2026-02-16T06:08:38.049854196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:52636"
level=debug ts=2026-02-16T06:08:58.898732837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:09:01.167561781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:09:08.050740591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:39970"
level=debug ts=2026-02-16T06:09:26.387602239Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:09:31.168984871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:09:35.746800139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:54962"
ts=2026-02-16T06:09:36.153504943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:59698"
level=debug ts=2026-02-16T06:09:50.927074643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:10:01.171119499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:10:06.155901995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:58244"
level=debug ts=2026-02-16T06:10:26.38721048Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=debug ts=2026-02-16T06:10:29.871621125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:10:31.173836815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:11:01.175374004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:11:08.05863478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:36174"
level=debug ts=2026-02-16T06:11:19.145037298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:11:19.147941979Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T06:11:26.38750785Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:11:31.17730961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:11:36.161951337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:54926"
ts=2026-02-16T06:12:01.179306953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:12:06.164464037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:36424"
level=debug ts=2026-02-16T06:12:21.054099366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:12:26.386727497Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:12:31.181819443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:12:35.757846642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:37478"
ts=2026-02-16T06:12:38.063950851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:45568"
level=debug ts=2026-02-16T06:12:55.759317732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:13:01.183945826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:13:26.386941048Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:13:31.185672908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:13:35.201615489Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:13:35.762330314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:55102"
ts=2026-02-16T06:14:01.187462188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:14:05.764403601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:59314"
ts=2026-02-16T06:14:06.172773447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:53144"
level=debug ts=2026-02-16T06:14:13.646690062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:14:26.387128321Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:14:31.189236389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:14:35.767021669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:46974"
ts=2026-02-16T06:15:01.190690427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:15:01.765956958Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:15:01.767514826Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:15:06.177315167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:53082"
level=debug ts=2026-02-16T06:15:26.386761028Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:15:31.192658519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:15:35.770261936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:35818"
ts=2026-02-16T06:15:36.179725314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:48948"
ts=2026-02-16T06:15:38.07567228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:40330"
level=debug ts=2026-02-16T06:15:39.659919896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:16:01.194396703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:16:19.507244104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:16:26.387057839Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:16:31.196204228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:16:35.774720136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:34046"
ts=2026-02-16T06:16:38.079023321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:35080"
ts=2026-02-16T06:17:01.198159009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:17:06.184616966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:39850"
ts=2026-02-16T06:17:08.080084644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:40828"
level=debug ts=2026-02-16T06:17:14.615561195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:17:26.387477663Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:17:31.200115255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:17:38.081115804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:49632"
ts=2026-02-16T06:18:01.201992317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:18:02.84343077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:18:05.780666715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:48580"
ts=2026-02-16T06:18:06.189048215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:45590"
level=debug ts=2026-02-16T06:18:26.386708246Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:18:31.203812039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:18:38.083893727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:41638"
ts=2026-02-16T06:19:01.204712073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T06:19:01.439523579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:19:01.441324061Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:19:08.084859391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:50728"
level=debug ts=2026-02-16T06:19:26.386960595Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:19:31.20659874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:19:40.515433017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:20:01.209116144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:20:08.088836495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:43708"
level=debug ts=2026-02-16T06:20:19.399330869Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:20:26.386805628Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:20:31.211237046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:20:55.441850345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:21:01.213427003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:21:06.200439291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:43964"
level=debug ts=2026-02-16T06:21:26.386934384Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=debug ts=2026-02-16T06:21:30.260046061Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:21:31.21522665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:21:38.093767919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:59692"
ts=2026-02-16T06:22:01.216676741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:22:05.794160183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:49710"
level=debug ts=2026-02-16T06:22:21.968703643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:22:21.970541607Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T06:22:26.386730415Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:22:31.218004503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:22:35.796547118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:43632"
ts=2026-02-16T06:22:36.206136236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:60382"
ts=2026-02-16T06:22:38.09726805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:42714"
level=debug ts=2026-02-16T06:22:57.391401496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:23:01.219566874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T06:23:26.386990401Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:23:31.224470376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:23:35.800418204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:42068"
ts=2026-02-16T06:23:38.100584819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:53966"
level=debug ts=2026-02-16T06:23:39.167871832Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:24:01.226856896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:24:24.465049048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:24:26.386514413Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:24:31.228806976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:24:35.804295483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:57828"
ts=2026-02-16T06:24:36.212768201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:48382"
ts=2026-02-16T06:25:01.230139223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T06:25:06.32512455Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:25:26.387078757Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:25:31.232381295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:25:35.807755945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:60396"
ts=2026-02-16T06:25:38.107768237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:55876"
ts=2026-02-16T06:26:01.234219101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T06:26:04.679021744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:26:04.681029354Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:26:08.108674892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:57526"
level=debug ts=2026-02-16T06:26:26.386846165Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:26:31.236210206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:26:35.810255061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:36788"
level=debug ts=2026-02-16T06:26:40.452629671Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:27:01.238684115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:27:08.112030331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:36390"
level=debug ts=2026-02-16T06:27:21.969541968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:27:26.386591814Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:27:31.241283065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:28:01.243655835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:28:22.113356567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:28:26.387357133Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:28:31.245300869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:28:35.817574321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:39334"
ts=2026-02-16T06:29:01.247438382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:29:17.990583778Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:29:26.387159947Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:29:31.248613811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:29:36.228803367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:53838"
level=debug ts=2026-02-16T06:29:54.647128547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:29:54.648871165Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:30:01.249950657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:30:06.23110143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:35790"
level=debug ts=2026-02-16T06:30:26.387391665Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:30:31.251110812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T06:30:39.17354491Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:31:01.253364177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:31:05.825317282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:57900"
ts=2026-02-16T06:31:08.128291491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:44984"
level=debug ts=2026-02-16T06:31:26.386962914Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:31:31.255605045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:31:36.236633368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:39596"
ts=2026-02-16T06:31:38.130120866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:35284"
level=debug ts=2026-02-16T06:31:40.690599778Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:32:01.257781085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:32:06.239261131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:35088"
ts=2026-02-16T06:32:08.131449397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:39958"
level=debug ts=2026-02-16T06:32:22.352606789Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:32:26.386807788Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:32:31.259447467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:33:01.26033331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T06:33:17.585712199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:33:26.386831102Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:33:31.262022007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:34:01.263539816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:34:05.839695456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:40226"
ts=2026-02-16T06:34:08.145733352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:49160"
level=debug ts=2026-02-16T06:34:14.805641761Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:34:14.807580832Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T06:34:26.387298842Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:34:31.264838847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:34:35.842088028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:57186"
ts=2026-02-16T06:34:38.14746214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:47958"
ts=2026-02-16T06:35:01.266435555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T06:35:06.544121433Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:35:26.38675844Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:35:31.26820501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:35:35.845869116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:60836"
ts=2026-02-16T06:35:36.250351048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:40068"
ts=2026-02-16T06:35:38.151521549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:58774"
level=debug ts=2026-02-16T06:35:42.637230921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:36:01.270224223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:36:26.38727116Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:36:31.271463013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:36:35.849516521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:35596"
level=debug ts=2026-02-16T06:36:40.444543873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:37:01.272805407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T06:37:26.38677041Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:37:31.274612332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:37:36.257698617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:46154"
level=debug ts=2026-02-16T06:37:38.660502884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:38:01.275851544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:38:05.854750024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:53648"
level=debug ts=2026-02-16T06:38:14.096725191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:38:14.098598101Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T06:38:26.3872717Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:38:31.277206691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:38:38.163260166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:40090"
ts=2026-02-16T06:39:01.279226394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:39:05.857165332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:38990"
level=debug ts=2026-02-16T06:39:08.322763704Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:39:26.38738825Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:39:31.280449594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:39:36.265557716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:55868"
level=debug ts=2026-02-16T06:39:52.604159031Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:40:01.282318211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:40:06.267475635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:36080"
level=debug ts=2026-02-16T06:40:26.387455858Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:40:31.284638531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:40:33.809665155Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:41:01.286882858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:41:05.864638859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:54130"
ts=2026-02-16T06:41:06.270709708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:33532"
ts=2026-02-16T06:41:08.174211969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:50364"
level=debug ts=2026-02-16T06:41:26.386871217Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=debug ts=2026-02-16T06:41:29.300753734Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:41:31.289100426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:42:01.291266685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:42:06.718136659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:42:06.719889087Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T06:42:26.386796484Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:42:31.29292366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:43:01.295202461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:43:05.639304068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:43:05.870127661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:44542"
level=debug ts=2026-02-16T06:43:26.386479725Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:43:31.297296635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:43:35.871908139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:39912"
ts=2026-02-16T06:43:38.184453951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:35160"
level=debug ts=2026-02-16T06:43:40.426086053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:44:01.298098687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:44:06.283314545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:34148"
level=debug ts=2026-02-16T06:44:26.386554102Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:44:31.300341694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:44:42.741703354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:45:01.302104411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:45:05.875837104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:42562"
ts=2026-02-16T06:45:06.287843307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:45296"
level=debug ts=2026-02-16T06:45:17.674633633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:45:26.386553787Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:45:31.304269324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:45:35.878249407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:46802"
ts=2026-02-16T06:45:38.193008703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:49874"
level=debug ts=2026-02-16T06:45:52.370695957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:45:52.372683832Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:46:01.306554647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:46:06.291542515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:44970"
ts=2026-02-16T06:46:08.194753969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:38008"
level=debug ts=2026-02-16T06:46:26.386835876Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:46:31.308325597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:46:35.882023637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:47290"
ts=2026-02-16T06:46:38.196463758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:58632"
level=debug ts=2026-02-16T06:46:49.129802887Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:47:01.310089016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:47:06.294776258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:36700"
level=debug ts=2026-02-16T06:47:24.458069348Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:47:26.387291383Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:47:31.312337313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:47:36.296683314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:37738"
ts=2026-02-16T06:48:01.314108281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:48:03.267661584Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:48:05.887847002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:57496"
level=debug ts=2026-02-16T06:48:26.387538228Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:48:31.316418917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:48:38.204262972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:36620"
level=debug ts=2026-02-16T06:48:45.283261426Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:49:01.317907353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:49:05.890331293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:33202"
level=debug ts=2026-02-16T06:49:26.07055945Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:49:26.072051602Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T06:49:26.386955932Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:49:31.319601431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:49:36.30440163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:60474"
ts=2026-02-16T06:50:01.321357463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:50:05.229290693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:50:06.306272817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:45932"
level=debug ts=2026-02-16T06:50:26.386872829Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
ts=2026-02-16T06:50:31.323327007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
ts=2026-02-16T06:50:35.896418601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:59594"
ts=2026-02-16T06:50:38.210240512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:50226"
level=debug ts=2026-02-16T06:50:45.165571718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:51:01.325360102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6332ef80 10.129.0.62:7946"
level=debug ts=2026-02-16T06:51:26.387028124Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1