level=info ts=2026-02-16T18:01:56.527767546Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T18:01:56.527799432Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-02-16T18:01:56.529714796Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T18:01:56.530916125Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T18:01:56.531018032Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.55 inf=eth0
level=info ts=2026-02-16T18:01:56.531653558Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-querier-77cddbcfd9-xg7mt-29464e37
ts=2026-02-16T18:01:56.531732804Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T18:01:56.531747199Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.55 advertisePort=7946
level=debug ts=2026-02-16T18:01:56.531765751Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.55 advertisePort=7946
level=info ts=2026-02-16T18:01:56.531908754Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-02-16T18:01:56.536265873Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T18:01:56.536291002Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
ts=2026-02-16T18:01:56.53687764Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:01:56.536901829Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:01:56.53691669Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.010192ms
level=error ts=2026-02-16T18:01:56.536934031Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T18:01:56.536944999Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T18:01:56.540417013Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:01:56.540438224Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T18:01:56.542062655Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:01:56.54210597Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T18:01:56.542356263Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=debug ts=2026-02-16T18:01:56.542702262Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=info ts=2026-02-16T18:01:56.542748022Z caller=worker.go:138 component=querier msg="Starting querier worker connected to query-scheduler" scheduler=loki-query-scheduler.product-kubearchive-logging.svc.cluster.local:9095
level=info ts=2026-02-16T18:01:56.561591829Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-02-16T18:01:56.561671538Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-02-16T18:01:56.561703403Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-02-16T18:01:56.561713933Z caller=module_service.go:82 msg=starting module=cache-generation-loader
level=debug ts=2026-02-16T18:01:56.561723665Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-02-16T18:01:56.561734198Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-02-16T18:01:56.561734885Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-02-16T18:01:56.561772093Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-02-16T18:01:56.561787468Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=memberlist-kv
level=debug ts=2026-02-16T18:01:56.56179714Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=ring
level=debug ts=2026-02-16T18:01:56.561813119Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=analytics
level=debug ts=2026-02-16T18:01:56.561827923Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ingester-querier
level=debug ts=2026-02-16T18:01:56.561909263Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=cache-generation-loader
level=debug ts=2026-02-16T18:01:56.561928044Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ingester-querier
level=debug ts=2026-02-16T18:01:56.561956722Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-02-16T18:01:56.561962428Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-02-16T18:01:56.561997451Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-02-16T18:01:56.562031358Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=runtime-config
level=debug ts=2026-02-16T18:01:56.562037211Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=server
level=info ts=2026-02-16T18:01:56.562042778Z caller=module_service.go:82 msg=starting module=ingester-querier
level=debug ts=2026-02-16T18:01:56.562053106Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=memberlist-kv
level=debug ts=2026-02-16T18:01:56.562057979Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ring
level=debug ts=2026-02-16T18:01:56.56206286Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=runtime-config
level=debug ts=2026-02-16T18:01:56.56206796Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=server
level=debug ts=2026-02-16T18:01:56.562073115Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=store
level=debug ts=2026-02-16T18:01:56.562078271Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-02-16T18:01:56.562084354Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ring
level=debug ts=2026-02-16T18:01:56.562088552Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-02-16T18:01:56.562093044Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-02-16T18:01:56.562097204Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-02-16T18:01:56.56210726Z caller=module_service.go:82 msg=starting module=querier
level=info ts=2026-02-16T18:01:56.562189247Z caller=loki.go:599 msg="Loki started" startup_time=101.475075ms
level=warn ts=2026-02-16T18:01:56.565845321Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-02-16T18:01:57.575626896Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:01:57.575657036Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:01:59.568357348Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-02-16T18:02:01.508717633Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:02:01.508742645Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T18:02:01.540195166Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:02:01.54022199Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-02-16T18:02:02.57121882Z caller=worker.go:235 component=querier msg="adding connection" addr=10.130.0.59:9095
level=error ts=2026-02-16T18:02:06.539755968Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:02:06.539785008Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T18:02:06.764186501Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:02:06.764215481Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T18:02:10.554125391Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T18:02:11.539127107Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:02:11.539152044Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=error ts=2026-02-16T18:02:16.540127571Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:02:16.540160512Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T18:02:18.613412017Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:02:18.613436855Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T18:02:30.700609527Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:02:34.069794193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:56548"
ts=2026-02-16T18:02:35.838857402Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.59:7946: dial tcp 10.128.0.59:7946: connect: connection refused"
ts=2026-02-16T18:02:35.83901421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.57:7946"
ts=2026-02-16T18:02:35.839587473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.55:7946"
ts=2026-02-16T18:02:35.839609314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:36592"
ts=2026-02-16T18:02:35.841119726Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.59:7946: dial tcp 10.130.0.59:7946: connect: connection refused"
ts=2026-02-16T18:02:35.842268183Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.62:7946: dial tcp 10.130.0.62:7946: connect: connection refused"
ts=2026-02-16T18:02:35.843300669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.61:7946"
level=info ts=2026-02-16T18:02:35.844598291Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=39.307646882s
ts=2026-02-16T18:02:43.434137521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:48194"
ts=2026-02-16T18:02:43.727581723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:54454"
level=debug ts=2026-02-16T18:02:48.472872008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:02:53.15393758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:02:56.537712997Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:03:12.31877674Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:03:13.226306272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:53986"
ts=2026-02-16T18:03:23.156148921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:03:44.970574282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T18:03:44.97222724Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:03:53.158688317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:03:54.661767572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60890"
level=debug ts=2026-02-16T18:03:56.538081822Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:04:00.872708069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:51896"
ts=2026-02-16T18:04:13.228818521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:59976"
ts=2026-02-16T18:04:23.161395629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:04:30.525005166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:04:30.876147105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:46854"
ts=2026-02-16T18:04:43.229948157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:58252"
ts=2026-02-16T18:04:53.163203187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:04:56.538726233Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:05:23.165724922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:05:35.856629931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:05:53.168348092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:05:54.670812536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:38482"
level=debug ts=2026-02-16T18:05:56.538257452Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:06:23.169681048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:06:53.17231505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:06:54.807944642Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T18:06:56.538556952Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:07:23.175603593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:07:24.67660506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:48652"
ts=2026-02-16T18:07:53.177904538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:07:54.677472352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:51986"
level=debug ts=2026-02-16T18:07:55.611099215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T18:07:56.538113822Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:08:00.897757638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:43944"
ts=2026-02-16T18:08:23.179610967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:08:24.679182517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:58280"
ts=2026-02-16T18:08:30.900647858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58708"
level=debug ts=2026-02-16T18:08:49.430648044Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T18:08:49.432352487Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:08:53.181237363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:08:54.680752299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:39348"
level=debug ts=2026-02-16T18:08:56.538233641Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:09:23.182736536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:09:43.25628945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:59702"
level=debug ts=2026-02-16T18:09:49.884512857Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:09:53.183764819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:09:56.538623588Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:10:13.257551796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:34600"
ts=2026-02-16T18:10:23.1858052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:10:43.259410726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:52040"
level=debug ts=2026-02-16T18:10:51.113161206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:10:53.188122863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:10:54.690835581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60350"
level=debug ts=2026-02-16T18:10:56.539121371Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:11:23.188964592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:11:24.693833823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41714"
ts=2026-02-16T18:11:30.919569332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:41674"
ts=2026-02-16T18:11:43.262234099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:58514"
level=debug ts=2026-02-16T18:11:47.459511481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:11:53.190083111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:11:56.53787008Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:12:23.191367068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:12:24.698939578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:38076"
ts=2026-02-16T18:12:30.925378525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:54746"
ts=2026-02-16T18:12:43.266526706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:35004"
level=debug ts=2026-02-16T18:12:46.892393213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:12:53.1938107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:12:56.537765409Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:13:23.197441425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:13:45.485527199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:13:45.487616774Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:13:53.200880565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:13:56.537932493Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:14:00.936246515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57832"
ts=2026-02-16T18:14:13.27088139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:44166"
ts=2026-02-16T18:14:23.203060284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:14:30.939438315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:56666"
ts=2026-02-16T18:14:43.272074838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:32834"
level=debug ts=2026-02-16T18:14:45.5004317Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:14:53.204772561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:14:56.5392775Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:15:23.207354643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:15:30.946376634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:51192"
level=debug ts=2026-02-16T18:15:42.828291113Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:15:53.210736496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:15:56.538664044Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:16:00.949436353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55144"
ts=2026-02-16T18:16:23.213227984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:16:24.722503548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:38424"
level=debug ts=2026-02-16T18:16:30.997608669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:16:43.280291689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:44030"
ts=2026-02-16T18:16:53.217842654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:16:56.537903931Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:17:00.955448968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45696"
ts=2026-02-16T18:17:13.281369643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:56292"
level=debug ts=2026-02-16T18:17:20.75114539Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:17:23.219834666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:17:24.727231661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:49494"
ts=2026-02-16T18:17:53.221230298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:17:54.728573371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:55682"
level=debug ts=2026-02-16T18:17:56.537876154Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:18:00.962217789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:43588"
level=debug ts=2026-02-16T18:18:18.040370446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:18:18.042182538Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:18:23.222723059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:18:24.729750165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:43098"
ts=2026-02-16T18:18:53.224436461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:18:56.538450442Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:19:00.96860266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:34696"
level=debug ts=2026-02-16T18:19:05.344974638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:19:23.225827861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:19:43.580790406Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:19:53.227171218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:19:56.538084208Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:20:13.294552365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:33928"
ts=2026-02-16T18:20:23.228462543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:20:42.762790298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:20:53.229368375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:20:54.740839293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42006"
level=debug ts=2026-02-16T18:20:56.538554921Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:21:21.671711963Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:21:23.232146909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:21:24.741714141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:38972"
ts=2026-02-16T18:21:30.984749057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:50522"
ts=2026-02-16T18:21:43.301631119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:56782"
ts=2026-02-16T18:21:53.234992061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:21:56.538778018Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:22:13.303075388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:48268"
level=debug ts=2026-02-16T18:22:16.049133002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:22:16.050751581Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:22:23.237557861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:22:30.996138727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47764"
ts=2026-02-16T18:22:53.239433499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:22:54.744339621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:58182"
level=debug ts=2026-02-16T18:22:56.537958435Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:22:58.018544915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:23:00.999401938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47816"
ts=2026-02-16T18:23:23.242345431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:23:51.335392632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:23:53.244921712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:23:56.537648144Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:24:01.006040797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:51298"
ts=2026-02-16T18:24:13.310670733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:58914"
ts=2026-02-16T18:24:23.246811214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:24:48.626392774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:24:53.248005984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:24:56.538267787Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:25:23.25032387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:25:27.102638466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:25:53.252405318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:25:56.538583052Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:26:23.254809195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:26:24.757260486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35514"
level=debug ts=2026-02-16T18:26:27.782245699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:26:27.784053989Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:26:53.256688476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:26:56.537707521Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:27:23.258188019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:27:24.053629631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:27:31.029204732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:48354"
ts=2026-02-16T18:27:43.325202801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:46314"
ts=2026-02-16T18:27:53.259844472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:27:56.537632112Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:28:04.676803883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:28:23.261300456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:28:31.035041923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:52002"
ts=2026-02-16T18:28:43.329900714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:59994"
ts=2026-02-16T18:28:53.264153943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:28:54.771173855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:57876"
level=debug ts=2026-02-16T18:28:56.538245366Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:29:01.038639792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55136"
level=debug ts=2026-02-16T18:29:06.957496815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:29:23.266383268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:29:43.565095593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:29:53.267599098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:29:54.775452568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:55528"
level=debug ts=2026-02-16T18:29:56.537574227Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:30:23.26872774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:30:32.151447609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:30:32.153036327Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:30:53.270127388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:30:56.538355805Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:31:23.27144896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:31:28.701490298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:31:43.343832456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:47594"
ts=2026-02-16T18:31:53.272506623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:31:56.538583594Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:32:15.495560446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:32:23.273439682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:32:43.346630043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:39852"
ts=2026-02-16T18:32:53.275007158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:32:56.538530773Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:33:01.979754038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:33:23.27669299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:33:37.411879178Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:33:53.278518375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:33:54.790572406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:55674"
level=debug ts=2026-02-16T18:33:56.538508029Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:34:13.349779636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:52862"
ts=2026-02-16T18:34:23.279674012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:34:31.074283397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:49184"
level=debug ts=2026-02-16T18:34:38.80185303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:34:38.803506446Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:34:53.28152794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:34:56.538167504Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:35:13.992244961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:35:23.284415184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:35:53.286269109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:35:56.538296212Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:35:59.609528982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:36:01.084757156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57036"
ts=2026-02-16T18:36:23.288283046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:36:31.087683327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:39870"
ts=2026-02-16T18:36:43.358095433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:38928"
ts=2026-02-16T18:36:53.291411375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:36:56.538171151Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:36:56.899494947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:37:01.09106582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:38454"
ts=2026-02-16T18:37:23.293654498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:37:31.094108067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58768"
ts=2026-02-16T18:37:53.295820328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:37:55.17263252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:37:56.538116055Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:38:01.09738967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:38774"
ts=2026-02-16T18:38:23.298380006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:38:24.809672532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60846"
ts=2026-02-16T18:38:31.100407946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:39082"
level=debug ts=2026-02-16T18:38:40.814879361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:38:40.816595417Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:38:53.300044402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:38:56.53805053Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:39:13.366535058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:59298"
ts=2026-02-16T18:39:23.30178447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:39:28.824741951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:39:31.106310284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:60244"
ts=2026-02-16T18:39:53.302763122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:39:56.537663121Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:40:01.109887149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:56570"
level=debug ts=2026-02-16T18:40:22.648828563Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:40:23.303640822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:40:24.817616066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:52916"
ts=2026-02-16T18:40:43.370615386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:42386"
ts=2026-02-16T18:40:53.30529421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:40:54.81887426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:39236"
level=debug ts=2026-02-16T18:40:56.538447314Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:41:21.117421515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:41:23.306695924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:41:53.308017368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:41:56.537998788Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:41:56.73291632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:42:23.309356777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:42:33.844072303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:42:33.845563912Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:42:53.311111876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:42:56.537892473Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:43:01.129184643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:42776"
level=debug ts=2026-02-16T18:43:21.291837901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:43:23.31390586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:43:43.38541991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:39220"
ts=2026-02-16T18:43:53.317567148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:43:56.538132561Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:44:08.515084765Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:44:23.320249575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:44:24.833748758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:51490"
ts=2026-02-16T18:44:31.138171984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:44520"
level=debug ts=2026-02-16T18:44:46.930952951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:44:53.323694778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:44:54.835706545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:48460"
level=debug ts=2026-02-16T18:44:56.538376031Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:45:01.141203447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:50950"
ts=2026-02-16T18:45:23.325701706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:45:43.392467947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:34772"
level=debug ts=2026-02-16T18:45:48.217771273Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:45:53.328178838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:45:54.839717592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:36286"
level=debug ts=2026-02-16T18:45:56.538302358Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:46:23.330565397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:46:30.09276033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:46:30.094661501Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:46:53.332011555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:46:56.538154569Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:47:17.764943599Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:47:23.333737751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:47:53.336544424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:47:53.974050835Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:47:56.538448519Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:48:01.160853843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:48100"
ts=2026-02-16T18:48:13.404783052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:34532"
ts=2026-02-16T18:48:23.337841201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:48:31.163588483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:59010"
ts=2026-02-16T18:48:53.340813354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:48:54.853402847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41010"
level=debug ts=2026-02-16T18:48:55.09865099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:48:56.538225006Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:49:01.167080945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:60342"
ts=2026-02-16T18:49:23.344125493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:49:31.170458932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:52716"
level=debug ts=2026-02-16T18:49:35.192546382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:49:43.410753958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:35810"
ts=2026-02-16T18:49:53.346132969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:49:54.858233793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46634"
level=debug ts=2026-02-16T18:49:56.538799913Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:50:16.268298903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:50:16.270164987Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:50:23.348959675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:50:24.859924453Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:34742"
ts=2026-02-16T18:50:53.350701847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:50:56.538441123Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:51:05.664693427Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:51:23.353310463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:51:24.862940314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47746"
ts=2026-02-16T18:51:31.183239179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:52634"
level=debug ts=2026-02-16T18:51:52.494616748Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:51:53.355711315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:51:56.537897853Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:52:13.419614829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:60684"
ts=2026-02-16T18:52:23.356895399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:52:24.865193479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:58926"
level=debug ts=2026-02-16T18:52:30.050144955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:52:31.188791244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55238"
ts=2026-02-16T18:52:43.421530538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:47616"
ts=2026-02-16T18:52:53.358454841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:52:56.538447963Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:53:22.575418298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:53:23.3609318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:53:31.196021393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:32946"
ts=2026-02-16T18:53:53.362948368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:53:54.870666761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:34258"
level=debug ts=2026-02-16T18:53:56.538021449Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:54:11.993877842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:54:11.99615613Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:54:23.364793505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:54:31.206070212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:41722"
ts=2026-02-16T18:54:43.4279967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:46258"
level=debug ts=2026-02-16T18:54:52.024393372Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:54:53.365845196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:54:56.538064817Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:55:13.429789232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:56794"
ts=2026-02-16T18:55:23.367047939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:55:40.158324185Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:55:53.368883837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:55:54.877534222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:39566"
level=debug ts=2026-02-16T18:55:56.538138225Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:56:23.372016788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:56:24.879198402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:49870"
level=debug ts=2026-02-16T18:56:38.353402425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:56:53.375088747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:56:56.538456192Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:57:01.223377445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:33920"
level=debug ts=2026-02-16T18:57:15.483459149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:57:23.378727801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:57:31.226221895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47476"
ts=2026-02-16T18:57:53.381202264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:57:56.538366502Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T18:58:11.911087727Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:58:11.91281336Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:58:23.383693433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:58:24.887772559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46630"
level=debug ts=2026-02-16T18:58:49.720879902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:58:53.386705535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:58:56.537773716Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T18:59:23.389155397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:59:34.411741698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:59:43.450582707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:41764"
ts=2026-02-16T18:59:53.390372326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:59:56.538067417Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T19:00:01.24178908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:59360"
ts=2026-02-16T19:00:13.452058691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:47254"
level=debug ts=2026-02-16T19:00:21.69602651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:00:23.39172978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T19:00:43.45338299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:43486"
ts=2026-02-16T19:00:53.392863023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T19:00:54.900022952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:53858"
level=debug ts=2026-02-16T19:00:56.538116066Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T19:01:21.929956247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:01:23.393988534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T19:01:53.394869321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T19:01:54.904793947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:59482"
level=debug ts=2026-02-16T19:01:56.538500154Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T19:02:03.035148317Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T19:02:03.036799483Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T19:02:23.39610756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T19:02:53.060655074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:02:53.397171697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T19:02:56.538408235Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T19:03:13.4641025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:57500"
ts=2026-02-16T19:03:23.398707764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T19:03:31.264358789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:37666"
level=debug ts=2026-02-16T19:03:41.678286519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:03:53.400674519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T19:03:56.538129607Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T19:04:13.467165148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:54048"
ts=2026-02-16T19:04:23.402712253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T19:04:43.036258282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:04:53.405352956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T19:04:54.914815026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:34800"
level=debug ts=2026-02-16T19:04:56.538720575Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T19:05:01.273515669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57998"
ts=2026-02-16T19:05:13.470786753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:41236"
level=debug ts=2026-02-16T19:05:18.123185829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:05:23.408586895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T19:05:53.412148598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T19:05:56.538591908Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T19:05:59.558655974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T19:05:59.560296832Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T19:06:23.415743066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T19:06:31.282124769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:34480"
ts=2026-02-16T19:06:43.477867162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:39442"
ts=2026-02-16T19:06:53.421601054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T19:06:56.537733349Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T19:06:58.582193733Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:07:01.285534211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45550"
ts=2026-02-16T19:07:13.479653337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:54986"
ts=2026-02-16T19:07:23.423655316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T19:07:24.924207909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41436"
ts=2026-02-16T19:07:43.481170599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:58594"
level=debug ts=2026-02-16T19:07:49.417875065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:07:53.42594529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T19:07:54.925536949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:58672"
level=debug ts=2026-02-16T19:07:56.538336657Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T19:08:01.291986952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:52076"
ts=2026-02-16T19:08:23.428356747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T19:08:29.993326815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:08:31.295532795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55960"
ts=2026-02-16T19:08:43.485586846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:54556"
ts=2026-02-16T19:08:53.430558436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T19:08:56.538051879Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
ts=2026-02-16T19:09:01.298396914Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:40580"
level=debug ts=2026-02-16T19:09:08.259035836Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:09:23.431935397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T19:09:43.488083447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:54262"
ts=2026-02-16T19:09:53.433718827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T19:09:54.934865389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:52146"
level=debug ts=2026-02-16T19:09:56.538532804Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-02-16T19:10:02.769942749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T19:10:02.771416586Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T19:10:23.436274578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T19:10:24.936765475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:48650"
level=debug ts=2026-02-16T19:10:46.909314388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"