level=info ts=2026-02-16T13:54:01.753910728Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T13:54:01.753954019Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-02-16T13:54:01.755662692Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T13:54:01.756540839Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T13:54:01.756641787Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.58 inf=eth0
level=info ts=2026-02-16T13:54:01.757195961Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-querier-77cddbcfd9-hb8bn-ba843256
ts=2026-02-16T13:54:01.757253673Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T13:54:01.757269279Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.58 advertisePort=7946
level=debug ts=2026-02-16T13:54:01.757293222Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.58 advertisePort=7946
level=info ts=2026-02-16T13:54:01.757429948Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-02-16T13:54:01.759527864Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T13:54:01.759563773Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
ts=2026-02-16T13:54:01.762175137Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T13:54:01.762192844Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T13:54:01.762201711Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=4.774174ms
level=error ts=2026-02-16T13:54:01.762208632Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T13:54:01.762219309Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T13:54:01.764318491Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:54:01.764334626Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T13:54:01.764528519Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:54:01.764546146Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T13:54:01.764855144Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=debug ts=2026-02-16T13:54:01.765157043Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=info ts=2026-02-16T13:54:01.765189997Z caller=worker.go:138 component=querier msg="Starting querier worker connected to query-scheduler" scheduler=loki-query-scheduler.product-kubearchive-logging.svc.cluster.local:9095
level=debug ts=2026-02-16T13:54:01.778664779Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=cache-generation-loader
level=debug ts=2026-02-16T13:54:01.778679119Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=memberlist-kv
level=info ts=2026-02-16T13:54:01.778689265Z caller=module_service.go:82 msg=starting module=cache-generation-loader
level=info ts=2026-02-16T13:54:01.778696585Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-02-16T13:54:01.778700179Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-02-16T13:54:01.778701345Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-02-16T13:54:01.778710745Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ingester-querier
level=debug ts=2026-02-16T13:54:01.778724207Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2026-02-16T13:54:01.778733421Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-02-16T13:54:01.778739336Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ingester-querier
level=info ts=2026-02-16T13:54:01.778758095Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T13:54:01.778772438Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=ring
level=debug ts=2026-02-16T13:54:01.77876175Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-02-16T13:54:01.778802204Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-02-16T13:54:01.779031899Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-02-16T13:54:01.779108413Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-02-16T13:54:01.779198427Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-02-16T13:54:01.779232074Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=runtime-config
level=debug ts=2026-02-16T13:54:01.779238214Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=server
level=info ts=2026-02-16T13:54:01.7792431Z caller=module_service.go:82 msg=starting module=ingester-querier
level=debug ts=2026-02-16T13:54:01.779253858Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-02-16T13:54:01.779258744Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ring
level=debug ts=2026-02-16T13:54:01.779263055Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=info ts=2026-02-16T13:54:01.779266985Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-02-16T13:54:01.779564288Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=memberlist-kv
level=debug ts=2026-02-16T13:54:01.779580183Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ring
level=debug ts=2026-02-16T13:54:01.779583378Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=runtime-config
level=debug ts=2026-02-16T13:54:01.779597784Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=server
level=debug ts=2026-02-16T13:54:01.779601424Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=store
level=debug ts=2026-02-16T13:54:01.779604183Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=analytics
level=info ts=2026-02-16T13:54:01.779607142Z caller=module_service.go:82 msg=starting module=querier
level=info ts=2026-02-16T13:54:01.779660063Z caller=loki.go:599 msg="Loki started" startup_time=94.360683ms
level=warn ts=2026-02-16T13:54:01.782896094Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-02-16T13:54:03.028180106Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:54:03.028208069Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T13:54:04.785097415Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-02-16T13:54:05.519078554Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:54:05.519366392Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T13:54:06.764138916Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:54:06.764172674Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-02-16T13:54:07.788017557Z caller=worker.go:235 component=querier msg="adding connection" addr=10.130.0.52:9095
ts=2026-02-16T13:54:09.793369405Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:54:09.793396775Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T13:54:11.764226133Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:54:11.764255264Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=error ts=2026-02-16T13:54:16.763786923Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:54:16.763812641Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T13:54:16.955397684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:54:21.916847673Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.61:7946: dial tcp 10.129.0.61:7946: connect: connection refused"
ts=2026-02-16T13:54:21.917772624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.60:7946"
ts=2026-02-16T13:54:21.918565146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.58:7946"
ts=2026-02-16T13:54:21.918612814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:38212"
ts=2026-02-16T13:54:21.921325201Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.69:7946: dial tcp 10.128.0.69:7946: connect: connection refused"
level=info ts=2026-02-16T13:54:21.921353479Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=20.159128977s
ts=2026-02-16T13:54:22.012752745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:38994"
ts=2026-02-16T13:54:22.027286292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:46206"
ts=2026-02-16T13:54:27.313274942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:36566"
level=debug ts=2026-02-16T13:54:35.977581701Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:54:55.795206664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T13:54:58.286424484Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T13:55:01.760962282Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T13:55:24.841183875Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:55:25.796268452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
ts=2026-02-16T13:55:27.236914123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:42506"
level=debug ts=2026-02-16T13:55:54.526441683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T13:55:54.52846931Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T13:55:55.797390144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T13:56:01.761338543Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T13:56:23.769478965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:36216"
ts=2026-02-16T13:56:25.798880737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T13:56:44.202901672Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:56:55.799982922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T13:57:01.761510047Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T13:57:21.046275142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:42722"
ts=2026-02-16T13:57:25.802089853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T13:57:53.773828614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:55176"
level=debug ts=2026-02-16T13:57:53.787181512Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:57:55.803423926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T13:57:57.245725617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:39764"
level=debug ts=2026-02-16T13:58:01.760856014Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T13:58:21.051466321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:33170"
ts=2026-02-16T13:58:23.775479201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:55050"
ts=2026-02-16T13:58:25.804719316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T13:58:49.892568579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:58:55.806776833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T13:59:01.760962935Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T13:59:21.055606453Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:36022"
ts=2026-02-16T13:59:25.808917834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T13:59:27.250118467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:43908"
ts=2026-02-16T13:59:51.057850935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:50054"
ts=2026-02-16T13:59:53.781017372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:52492"
ts=2026-02-16T13:59:55.810991296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T13:59:57.328334376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T14:00:01.760879948Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:00:25.812602125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:00:27.252637022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:36444"
ts=2026-02-16T14:00:53.784036464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:37324"
ts=2026-02-16T14:00:55.814541498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:00:57.254261385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:43696"
level=debug ts=2026-02-16T14:01:01.760686716Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:01:08.632074098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T14:01:08.634013891Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:01:25.816569295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:01:46.833706787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:01:53.787234671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:56860"
ts=2026-02-16T14:01:55.818354138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:01:57.257867472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:43836"
level=debug ts=2026-02-16T14:02:01.760492481Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:02:23.788697756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:43428"
ts=2026-02-16T14:02:25.819770327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:02:34.255325657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:02:51.071923518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:47680"
ts=2026-02-16T14:02:55.821193207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:02:57.260321738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:41480"
level=debug ts=2026-02-16T14:03:01.761096394Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:03:11.706119369Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:03:21.074133879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:45444"
ts=2026-02-16T14:03:25.823065504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:03:55.627019266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:03:55.82487382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:03:57.263055481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:55144"
level=debug ts=2026-02-16T14:04:01.760523708Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:04:25.826192974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:04:37.027537166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:04:37.029417402Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:04:55.827837585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:05:01.7614843Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:05:19.663262138Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:05:25.82942008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
ts=2026-02-16T14:05:55.83078307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:06:01.761502901Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:06:13.509322268Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:06:21.087571949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:53828"
ts=2026-02-16T14:06:23.803215399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:48254"
ts=2026-02-16T14:06:25.83192366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:06:50.502631266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:06:51.090097312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:39816"
ts=2026-02-16T14:06:55.833811768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:07:01.760681969Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:07:21.092229316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:54504"
ts=2026-02-16T14:07:23.806930092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:57034"
ts=2026-02-16T14:07:25.835017988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:07:34.789262954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:07:51.098788516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56340"
ts=2026-02-16T14:07:55.836215262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:08:01.761147984Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:08:21.108000999Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:49114"
ts=2026-02-16T14:08:25.837667765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:08:33.863630862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:08:33.865506309Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:08:51.110901816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:57220"
ts=2026-02-16T14:08:53.814782424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:53878"
ts=2026-02-16T14:08:55.838882824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:09:01.761135962Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:09:22.144396458Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:09:25.841264625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:09:55.843804254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:09:57.282925567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:53888"
level=debug ts=2026-02-16T14:10:00.352506953Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:10:01.760486657Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:10:25.845861223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:10:47.595662378Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:10:53.821118644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:39916"
ts=2026-02-16T14:10:55.847416204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:10:57.285516877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:36500"
level=debug ts=2026-02-16T14:11:01.761516498Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:11:25.84865832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:11:42.485573369Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:11:51.124034691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:37796"
ts=2026-02-16T14:11:53.824762576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:33252"
ts=2026-02-16T14:11:55.85071369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:12:01.761094156Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:12:21.126139376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:54444"
ts=2026-02-16T14:12:25.85296423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:12:42.389174088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:12:42.390972255Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:12:53.827975747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:46110"
ts=2026-02-16T14:12:55.854512627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
ts=2026-02-16T14:12:57.299988436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:37866"
level=debug ts=2026-02-16T14:13:01.760654173Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:13:21.129974735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:53270"
ts=2026-02-16T14:13:25.855558545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:13:35.438026504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:13:55.857371248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:14:01.760619165Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:14:23.832285831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:44642"
ts=2026-02-16T14:14:25.859302408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:14:30.27806232Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:14:55.860912875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:15:01.760490922Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:15:23.836899738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:38800"
ts=2026-02-16T14:15:25.862438325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:15:27.308389382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:58506"
level=debug ts=2026-02-16T14:15:31.802564564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:15:51.141839842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:45036"
ts=2026-02-16T14:15:55.864335927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:16:01.760498325Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:16:25.865855544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:16:27.406229516Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:16:55.867727141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:17:01.761422598Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:17:08.604759301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:17:08.60654132Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:17:25.868910346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:17:50.026907719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:17:55.870598014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:18:01.760661156Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:18:21.152422724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:55830"
ts=2026-02-16T14:18:25.872240747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:18:27.319689828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:39546"
level=debug ts=2026-02-16T14:18:27.692130802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:18:55.873905994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:19:01.760671754Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:19:12.392221447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:19:21.156184409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:35048"
ts=2026-02-16T14:19:25.875353713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
ts=2026-02-16T14:19:27.322730673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:41204"
ts=2026-02-16T14:19:51.158760509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56470"
ts=2026-02-16T14:19:55.876183491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:20:01.761598106Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:20:03.35135907Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:20:25.87752575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
ts=2026-02-16T14:20:27.326392785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:37920"
ts=2026-02-16T14:20:53.857005684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:60188"
ts=2026-02-16T14:20:55.878970582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:20:57.455123373Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:20:57.456828985Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T14:21:01.761017832Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:21:25.88051747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:21:27.329341148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:55540"
level=debug ts=2026-02-16T14:21:53.996082124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:21:55.882647853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:22:01.760562375Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:22:23.863082516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:52662"
ts=2026-02-16T14:22:25.884648353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:22:51.172055768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:39046"
level=debug ts=2026-02-16T14:22:51.756969368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:22:55.886632365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:23:01.760994444Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:23:21.17452381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:43294"
ts=2026-02-16T14:23:23.86643788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:36560"
ts=2026-02-16T14:23:25.88836909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:23:49.904540218Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:23:55.889953832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:24:01.761163867Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:24:23.868987621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:43836"
ts=2026-02-16T14:24:25.891121235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:24:48.506794076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:24:55.892293913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:25:01.760499675Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:25:25.893703445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:25:42.402307629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:25:42.404083083Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:25:51.185764938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:54918"
ts=2026-02-16T14:25:53.873921542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:50692"
ts=2026-02-16T14:25:55.895637926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
ts=2026-02-16T14:25:57.345826134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:45032"
level=debug ts=2026-02-16T14:26:01.761125955Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:26:25.896956503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:26:28.001006072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:26:53.877123952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:33174"
ts=2026-02-16T14:26:55.899581586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:27:01.761231554Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:27:21.192008224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:46860"
ts=2026-02-16T14:27:23.878306068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:43318"
ts=2026-02-16T14:27:25.901274464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:27:26.244980524Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:27:51.194201819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:60422"
ts=2026-02-16T14:27:53.879467601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:54684"
ts=2026-02-16T14:27:55.903046675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:27:57.352563143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:58030"
level=debug ts=2026-02-16T14:28:01.760760764Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:28:16.584669548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:28:25.904686563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
ts=2026-02-16T14:28:53.8828764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:54872"
level=debug ts=2026-02-16T14:28:55.060932075Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:28:55.906392636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:29:01.761206278Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:29:23.883938273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:47634"
ts=2026-02-16T14:29:25.907916367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:29:42.123456451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:29:42.125158075Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:29:55.909604897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:30:01.761035584Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:30:25.910832076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:30:26.967163995Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:30:27.360947169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:46726"
ts=2026-02-16T14:30:55.911890559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:31:01.76066352Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:31:10.528824834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:31:25.91356481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:31:51.216120516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:58074"
ts=2026-02-16T14:31:55.914913042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:31:57.803188968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:32:01.761182326Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:32:23.89747352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:45068"
ts=2026-02-16T14:32:25.916373243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:32:39.466154917Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:32:53.898879081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:44904"
ts=2026-02-16T14:32:55.918007878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:32:57.367367333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:51626"
level=debug ts=2026-02-16T14:33:01.761153675Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:33:23.900038608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:47076"
ts=2026-02-16T14:33:25.919309553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:33:27.369201918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:41732"
level=debug ts=2026-02-16T14:33:27.603663329Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:33:27.605298975Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:33:51.225044794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:47800"
ts=2026-02-16T14:33:55.920631376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:33:57.370492582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:50916"
level=debug ts=2026-02-16T14:34:01.761493315Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:34:05.25916493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:34:21.22719593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:47210"
ts=2026-02-16T14:34:23.902502855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:55378"
ts=2026-02-16T14:34:25.922471871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:34:42.90595094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:34:51.229063131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:47444"
ts=2026-02-16T14:34:53.904238404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:39412"
ts=2026-02-16T14:34:55.924757223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:35:01.760928103Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:35:20.895953083Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:35:25.926367039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:35:51.233469278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:57436"
ts=2026-02-16T14:35:55.927916347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:35:57.379617465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:48492"
level=debug ts=2026-02-16T14:36:01.760641199Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:36:21.842898213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:36:25.930105057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:36:53.911472405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:58018"
ts=2026-02-16T14:36:55.932121882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:36:57.382727576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:59092"
level=debug ts=2026-02-16T14:37:01.76060395Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:37:11.933755034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:37:11.935799324Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:37:23.913173332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:49672"
ts=2026-02-16T14:37:25.933642442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
ts=2026-02-16T14:37:55.934858694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:38:01.760842548Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:38:06.183046704Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:38:23.916722489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:40914"
ts=2026-02-16T14:38:25.936495888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:38:27.387972964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:35496"
ts=2026-02-16T14:38:51.245126711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:60664"
level=debug ts=2026-02-16T14:38:51.787214901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:38:55.938565449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:39:01.761187945Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:39:21.247588152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:35858"
ts=2026-02-16T14:39:25.940296663Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:39:33.529588922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:39:51.249938293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:33462"
ts=2026-02-16T14:39:53.922513349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:56678"
ts=2026-02-16T14:39:55.941390005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:40:01.760881971Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:40:23.044159253Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:40:23.924006225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:35754"
ts=2026-02-16T14:40:25.943130845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:40:51.254183702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:58856"
ts=2026-02-16T14:40:55.944326606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:40:57.397240461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:47766"
level=debug ts=2026-02-16T14:41:01.761142646Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:41:21.256019517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:60128"
ts=2026-02-16T14:41:23.927692635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:59816"
level=debug ts=2026-02-16T14:41:24.579380636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:41:24.581093999Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:41:25.94601935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:41:51.257911142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:48404"
ts=2026-02-16T14:41:53.929165023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:47694"
ts=2026-02-16T14:41:55.947463743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:41:57.401221147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:56240"
level=debug ts=2026-02-16T14:42:01.761001565Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:42:20.721453272Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:42:25.949128727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:42:55.950850832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:43:01.760865954Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:43:09.562051195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:43:21.264994659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:51106"
ts=2026-02-16T14:43:25.952644271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:43:53.936262943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:34078"
ts=2026-02-16T14:43:55.954067107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:43:57.410323052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:44060"
level=debug ts=2026-02-16T14:44:01.760901769Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:44:04.534514645Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:44:23.937707451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:46420"
ts=2026-02-16T14:44:25.956713106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:44:51.27188574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:47454"
ts=2026-02-16T14:44:55.958060219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:45:01.761480352Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:45:02.237969598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:45:21.274126895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:48720"
ts=2026-02-16T14:45:23.940071693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:40990"
ts=2026-02-16T14:45:25.959309401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
ts=2026-02-16T14:45:53.941059465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:34644"
ts=2026-02-16T14:45:55.960672328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:46:00.714869695Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:46:00.716633109Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T14:46:01.761087944Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:46:23.942119369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:39694"
ts=2026-02-16T14:46:25.961772851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:46:46.292347639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:46:53.94363607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:37946"
ts=2026-02-16T14:46:55.963609768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
ts=2026-02-16T14:46:57.420325439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:47326"
level=debug ts=2026-02-16T14:47:01.761559836Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:47:21.282036084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:47258"
ts=2026-02-16T14:47:23.94499096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:52462"
level=debug ts=2026-02-16T14:47:24.819358596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:47:25.965384018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
ts=2026-02-16T14:47:55.967545672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef36c79 10.129.0.59:7946"
level=debug ts=2026-02-16T14:48:01.760691109Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
ts=2026-02-16T14:48:25.968814344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:48:26.362126202Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:48:27.425870584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:34940"