level=info ts=2026-05-04T22:49:43.252828054Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-04T22:49:43.252881505Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-04T22:49:43.255593358Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-04T22:49:43.256171182Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-04T22:49:43.256303027Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.85 inf=eth0
level=info ts=2026-05-04T22:49:43.257009812Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-querier-5586f77b45-626pb-c275cc1c
ts=2026-05-04T22:49:43.257150643Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-04T22:49:43.257183421Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.85 advertisePort=7946
level=debug ts=2026-05-04T22:49:43.257213326Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.85 advertisePort=7946
level=info ts=2026-05-04T22:49:43.257432921Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-05-04T22:49:43.260894126Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T22:49:43.260958237Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
ts=2026-05-04T22:49:43.263570111Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-04T22:49:43.263661475Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-04T22:49:43.263706164Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.274597ms
level=error ts=2026-05-04T22:49:43.263730651Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-04T22:49:43.263748758Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-04T22:49:43.266251375Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:49:43.266297947Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-04T22:49:43.26642677Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:49:43.266451773Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-05-04T22:49:43.266986043Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=debug ts=2026-05-04T22:49:43.267414368Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=info ts=2026-05-04T22:49:43.267466068Z caller=worker.go:138 component=querier msg="Starting querier worker connected to query-scheduler" scheduler=loki-query-scheduler.product-kubearchive-logging.svc.cluster.local:9095
level=debug ts=2026-05-04T22:49:43.285902249Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ring
level=info ts=2026-05-04T22:49:43.285931707Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-04T22:49:43.285940282Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-04T22:49:43.285929761Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-04T22:49:43.285983172Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=memberlist-kv
level=debug ts=2026-05-04T22:49:43.285981312Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-04T22:49:43.285991764Z caller=module_service.go:82 msg=starting module=cache-generation-loader
level=debug ts=2026-05-04T22:49:43.286010298Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ring
level=debug ts=2026-05-04T22:49:43.286012392Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-05-04T22:49:43.286017617Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-04T22:49:43.286054353Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=ring
level=debug ts=2026-05-04T22:49:43.286054009Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-05-04T22:49:43.286714165Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-04T22:49:43.286723207Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-05-04T22:49:43.286763007Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-05-04T22:49:43.286796482Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=runtime-config
level=debug ts=2026-05-04T22:49:43.286800785Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=server
level=info ts=2026-05-04T22:49:43.286804112Z caller=module_service.go:82 msg=starting module=ingester-querier
level=debug ts=2026-05-04T22:49:43.28681248Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=runtime-config
level=debug ts=2026-05-04T22:49:43.28681657Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=server
level=debug ts=2026-05-04T22:49:43.286819728Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=store
level=debug ts=2026-05-04T22:49:43.286824015Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-05-04T22:49:43.28682663Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2026-05-04T22:49:43.286829956Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ingester-querier
level=debug ts=2026-05-04T22:49:43.28683259Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=info ts=2026-05-04T22:49:43.286835858Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-05-04T22:49:43.286850429Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=analytics
level=debug ts=2026-05-04T22:49:43.286858191Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=cache-generation-loader
level=debug ts=2026-05-04T22:49:43.286862053Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ingester-querier
level=debug ts=2026-05-04T22:49:43.28686564Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=memberlist-kv
level=info ts=2026-05-04T22:49:43.28686894Z caller=module_service.go:82 msg=starting module=querier
level=info ts=2026-05-04T22:49:43.286922516Z caller=loki.go:615 msg="Loki started" startup_time=108.105098ms
level=info ts=2026-05-04T22:49:43.29039846Z caller=worker.go:235 component=querier msg="adding connection" addr=172.30.100.240:9095
ts=2026-05-04T22:49:45.091315246Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:49:45.091343146Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:49:48.2458921Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:49:48.245915235Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-04T22:49:48.265323941Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:49:48.265351078Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
ts=2026-05-04T22:49:52.473401614Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:49:52.473424593Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-04T22:50:02.851971294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:50:06.062545788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:44078"
ts=2026-05-04T22:50:08.065163025Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.63:7946: dial tcp 10.129.0.63:7946: connect: connection refused"
ts=2026-05-04T22:50:08.066335223Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.62:7946: dial tcp 10.129.0.62:7946: connect: connection refused"
ts=2026-05-04T22:50:08.066415456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.85:7946"
ts=2026-05-04T22:50:08.066442475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:40674"
ts=2026-05-04T22:50:08.067991689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.64:7946"
level=info ts=2026-05-04T22:50:08.068755212Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=24.804941259s
ts=2026-05-04T22:50:08.605035043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:52652"
ts=2026-05-04T22:50:09.241118432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:39606"
ts=2026-05-04T22:50:19.188144254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T22:50:21.955262861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:50:24.66837042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:40152"
level=debug ts=2026-05-04T22:50:41.635386896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-04T22:50:43.26331417Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T22:50:49.190359623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T22:51:05.712204418Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:51:19.19201997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T22:51:24.671407492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:36350"
level=debug ts=2026-05-04T22:51:31.812421086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-04T22:51:31.814018803Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T22:51:43.262654409Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T22:51:49.194201337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T22:52:06.39552716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:52:19.196406364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T22:52:43.262323292Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T22:52:45.140772288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:44916"
ts=2026-05-04T22:52:49.198817003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T22:53:16.779197224Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:53:19.200916227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T22:53:43.262710411Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T22:53:49.203590655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T22:54:09.305293398Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:54:19.205840648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T22:54:25.277135943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:47544"
level=debug ts=2026-05-04T22:54:43.263037173Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T22:54:49.208115271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T22:54:54.681595212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59902"
level=debug ts=2026-05-04T22:55:11.306178242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:55:19.210759213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T22:55:24.683481701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53506"
level=debug ts=2026-05-04T22:55:43.262545951Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T22:55:45.149964879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:54378"
ts=2026-05-04T22:55:49.212529775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T22:56:15.151909404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:55438"
ts=2026-05-04T22:56:19.214243907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T22:56:24.046132884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-04T22:56:24.047720723Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T22:56:25.28373534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:52862"
level=debug ts=2026-05-04T22:56:43.26232497Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T22:56:49.215987759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T22:56:54.689826248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:38038"
ts=2026-05-04T22:57:15.155461944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:38096"
level=debug ts=2026-05-04T22:57:18.724672286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T22:57:19.217590217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T22:57:43.262637873Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T22:57:49.220323944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T22:57:57.239548872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T22:58:19.221806626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T22:58:25.288902316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:48864"
level=debug ts=2026-05-04T22:58:33.616842498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T22:58:43.263383994Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T22:58:49.224131263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T22:58:55.291206736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:58096"
level=debug ts=2026-05-04T22:59:09.845279245Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T22:59:19.226568042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T22:59:24.69739664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:52310"
ts=2026-05-04T22:59:25.292674763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:39598"
level=debug ts=2026-05-04T22:59:43.262726051Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T22:59:49.228766807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T22:59:49.949710993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T22:59:49.95157816Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T22:59:54.699564866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:52332"
ts=2026-05-04T22:59:55.295432319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:55098"
ts=2026-05-04T23:00:15.165011771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:46786"
ts=2026-05-04T23:00:19.231173452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:00:39.610865946Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:00:43.262467179Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:00:49.232754531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:01:15.169377662Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49350"
ts=2026-05-04T23:01:19.235145413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:01:24.705568795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:57624"
level=debug ts=2026-05-04T23:01:37.176660139Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:01:43.262759405Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:01:45.170992481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:41936"
ts=2026-05-04T23:01:49.237587127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:01:54.707561001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:33508"
ts=2026-05-04T23:01:55.301719518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:58206"
ts=2026-05-04T23:02:19.239190355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:02:19.904794243Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:02:43.262356574Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:02:45.175397355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:55620"
ts=2026-05-04T23:02:49.241473969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:03:11.998748851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:03:19.243071645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:03:24.713330893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53240"
ts=2026-05-04T23:03:25.306959275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:52908"
level=debug ts=2026-05-04T23:03:43.263247771Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:03:49.245345661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:03:52.714358088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:03:52.716384208Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:04:15.178892382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:50698"
ts=2026-05-04T23:04:19.247603931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:04:33.573706698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:04:43.262704927Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:04:45.180600608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60254"
ts=2026-05-04T23:04:49.249317254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:04:54.720020456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46016"
ts=2026-05-04T23:04:55.311353228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:40714"
ts=2026-05-04T23:05:15.183229039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49104"
ts=2026-05-04T23:05:19.251435149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:05:20.180877495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:05:24.721970812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53328"
level=debug ts=2026-05-04T23:05:43.263181344Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:05:49.253942821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:05:54.725115232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:58294"
level=debug ts=2026-05-04T23:06:13.477792343Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:06:19.255581315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:06:25.317619899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:51566"
level=debug ts=2026-05-04T23:06:43.264175621Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:06:49.257400312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:07:02.786897169Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:07:19.263456625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:07:24.730782577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:36454"
level=debug ts=2026-05-04T23:07:43.26272477Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:07:49.265962254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:07:52.228442234Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:07:52.230893469Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:08:19.2682824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:08:25.323246289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:47482"
level=debug ts=2026-05-04T23:08:43.266277727Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-05-04T23:08:48.091820335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:08:49.270672536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:08:54.737451787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:34582"
ts=2026-05-04T23:09:19.273735355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:09:30.173136566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:09:43.265353381Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:09:49.276138731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:10:19.278130718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:10:22.841296208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:10:25.329386298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:39844"
level=debug ts=2026-05-04T23:10:43.265380075Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:10:45.201872817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:55066"
ts=2026-05-04T23:10:49.280124191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:10:59.837993217Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:11:15.204019179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:46568"
ts=2026-05-04T23:11:19.28225125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:11:24.747698138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:47330"
level=debug ts=2026-05-04T23:11:39.864792542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:11:39.866860644Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T23:11:43.26259168Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:11:49.284829214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:11:55.333894828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:60688"
ts=2026-05-04T23:12:19.287720749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:12:24.751975792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55244"
level=debug ts=2026-05-04T23:12:25.812419469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:12:43.270922883Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:12:49.293046179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:13:19.297784627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:13:25.855176787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:13:43.265271281Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:13:49.300100741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:13:54.756624833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:33054"
ts=2026-05-04T23:13:55.340956563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:43820"
ts=2026-05-04T23:14:19.302332634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:14:24.514921568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:14:24.758526222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:35712"
level=debug ts=2026-05-04T23:14:43.262739531Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:14:45.2150387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:35026"
ts=2026-05-04T23:14:49.304727199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:15:08.645778746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:15:15.217605174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:46148"
ts=2026-05-04T23:15:19.306790464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:15:25.345950642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:48546"
level=debug ts=2026-05-04T23:15:43.262498194Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-05-04T23:15:46.727826684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:15:46.729919195Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:15:49.308654371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:15:54.763752829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55368"
ts=2026-05-04T23:16:19.310568516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:16:31.482858121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:16:43.262565216Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:16:45.223346445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:44598"
ts=2026-05-04T23:16:49.3126608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:16:54.767843366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53520"
ts=2026-05-04T23:16:55.351449572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:59114"
ts=2026-05-04T23:17:19.314856595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:17:24.769746621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:58530"
level=debug ts=2026-05-04T23:17:31.564313041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:17:43.262778413Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:17:49.317276901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:17:54.77218281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:40550"
level=debug ts=2026-05-04T23:18:05.75612247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:18:15.229403298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:56880"
ts=2026-05-04T23:18:19.319888144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:18:24.77391193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:50234"
level=debug ts=2026-05-04T23:18:43.089720089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:18:43.263265643Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:18:45.231628496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:53160"
ts=2026-05-04T23:18:49.322182166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:18:54.776602321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59130"
ts=2026-05-04T23:19:19.323816688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:19:25.357975272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:42502"
level=debug ts=2026-05-04T23:19:38.548507382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:19:38.550215163Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T23:19:43.27608311Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:19:45.235082833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:35390"
ts=2026-05-04T23:19:49.326192454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:20:19.328013941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:20:24.781729656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55226"
level=debug ts=2026-05-04T23:20:25.267552895Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:20:43.263413524Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:20:49.33049106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:21:15.238447511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60034"
ts=2026-05-04T23:21:19.332191079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:21:24.065612165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:21:24.784361597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:48468"
ts=2026-05-04T23:21:25.363601871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:34108"
level=debug ts=2026-05-04T23:21:43.263767392Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:21:49.334822654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:22:08.389318882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:22:19.336342154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:22:25.367311201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:55362"
level=debug ts=2026-05-04T23:22:43.262963933Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:22:45.243463281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:56002"
ts=2026-05-04T23:22:49.338295529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:23:07.56142535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:23:19.340260792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:23:24.79075491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:48676"
level=debug ts=2026-05-04T23:23:43.263043592Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-05-04T23:23:43.570052131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:23:43.572072059Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:23:49.342312762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:23:55.373861947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:43918"
ts=2026-05-04T23:24:15.248537111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:43096"
ts=2026-05-04T23:24:19.344058219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:24:25.37556174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:43460"
level=debug ts=2026-05-04T23:24:33.646463447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:24:43.263343003Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:24:49.345733565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:25:17.584300358Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:25:19.347900281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:25:43.263354713Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:25:49.350365998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:26:06.194887719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:26:19.352457433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:26:43.26221079Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:26:49.3544645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:26:54.801895274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46832"
level=debug ts=2026-05-04T23:27:08.528551182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:27:19.356407737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:27:24.806769875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53046"
level=debug ts=2026-05-04T23:27:43.262281548Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:27:49.358216704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:27:55.395621689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:39210"
level=debug ts=2026-05-04T23:27:57.382655104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:27:57.384158255Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:28:19.360139969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:28:25.413662751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:52312"
level=debug ts=2026-05-04T23:28:43.262620511Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:28:49.362149613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:28:51.142975031Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:28:54.817711691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:43336"
ts=2026-05-04T23:29:15.262794022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:43790"
ts=2026-05-04T23:29:19.363762096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:29:25.417994331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:34640"
level=debug ts=2026-05-04T23:29:29.632988156Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:29:43.262964394Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:29:49.366161172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:30:19.368357816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:30:25.004465627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:30:43.263354804Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:30:49.370135403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:31:09.520290559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:31:15.269131277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60694"
ts=2026-05-04T23:31:19.372036673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:31:43.262832682Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:31:49.374389789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:31:54.828241966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:51318"
ts=2026-05-04T23:31:55.42723094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:56924"
level=debug ts=2026-05-04T23:32:06.414600061Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:32:06.416327128Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:32:19.376123341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:32:25.429882223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:34454"
level=debug ts=2026-05-04T23:32:43.262465817Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:32:49.378548029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:32:55.508955913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:33:15.275816976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:56572"
ts=2026-05-04T23:33:19.380872699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:33:24.833512412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:36846"
ts=2026-05-04T23:33:25.434394438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:40472"
level=debug ts=2026-05-04T23:33:38.150397962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:33:43.263447863Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:33:45.277755421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:59240"
ts=2026-05-04T23:33:49.383463326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:34:19.385902967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:34:33.517553087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:34:43.268714909Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:34:45.281879578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:42482"
ts=2026-05-04T23:34:49.389621664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:34:54.839286906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:39356"
ts=2026-05-04T23:34:55.439815372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:48882"
ts=2026-05-04T23:35:19.392394926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:35:27.325489864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:35:43.272571679Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:35:49.394927576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:36:15.286624706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:38858"
ts=2026-05-04T23:36:19.396880308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:36:25.024973388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:36:25.026671542Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:36:25.444704301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:59246"
level=debug ts=2026-05-04T23:36:43.263054277Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:36:49.399064892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:37:03.341594364Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:37:19.400927255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:37:43.262195627Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:37:45.29184497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:47534"
ts=2026-05-04T23:37:49.403323862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:37:55.449292684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:53482"
level=debug ts=2026-05-04T23:38:03.999889338Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:38:19.405449885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:38:43.26267753Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:38:49.407572958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:38:53.304876935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:39:19.409522227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:39:24.852463693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53884"
level=debug ts=2026-05-04T23:39:31.742562526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:39:43.263122156Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:39:49.412170524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:39:55.455031946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:53316"
level=debug ts=2026-05-04T23:40:08.305045485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:40:08.30652766Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:40:15.300968616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:42658"
ts=2026-05-04T23:40:19.414761567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:40:24.855723283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:33106"
ts=2026-05-04T23:40:25.45681028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:53010"
level=debug ts=2026-05-04T23:40:43.263017009Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-05-04T23:40:47.064101201Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:40:49.417117091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:41:19.419623433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:41:23.150196176Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:41:24.858647846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:35748"
ts=2026-05-04T23:41:25.459509064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:36574"
level=debug ts=2026-05-04T23:41:43.263284497Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:41:45.305703009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:41044"
ts=2026-05-04T23:41:49.421847852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:42:11.16347923Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:42:15.307714983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:54520"
ts=2026-05-04T23:42:19.424285746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:42:24.86229798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:44364"
ts=2026-05-04T23:42:25.462764607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:48424"
level=debug ts=2026-05-04T23:42:43.263046471Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:42:49.425855264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:42:55.464964437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:47684"
level=debug ts=2026-05-04T23:43:05.34462758Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:43:19.427882206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:43:43.263360037Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-05-04T23:43:44.712249208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:43:44.713941718Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:43:49.430221672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:44:15.314219044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:34766"
ts=2026-05-04T23:44:19.431949706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:44:19.891243797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:44:24.868530893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:43028"
level=debug ts=2026-05-04T23:44:43.263142794Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:44:49.434533124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:44:54.870782163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46682"
level=debug ts=2026-05-04T23:44:56.071335988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:45:19.43647553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:45:43.262660478Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:45:45.319525978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:56718"
ts=2026-05-04T23:45:49.439010674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:45:52.591633871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:46:19.441297776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:46:43.262613532Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-05-04T23:46:48.906119511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:46:49.443211272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:47:19.445430348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:47:24.879460791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:44832"
ts=2026-05-04T23:47:25.480445567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:53330"
level=debug ts=2026-05-04T23:47:42.178303762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:47:42.180102048Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T23:47:43.262724536Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:47:45.326811955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60816"
ts=2026-05-04T23:47:49.447966081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:47:55.482475323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:56696"
ts=2026-05-04T23:48:19.449767215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:48:30.811494435Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:48:43.263040394Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:48:49.451757891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:48:55.486418628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:48140"
level=debug ts=2026-05-04T23:49:05.222810214Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:49:19.453794044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:49:24.886460279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:42052"
level=debug ts=2026-05-04T23:49:43.262386932Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:49:49.455586294Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:49:54.888824722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:36020"
ts=2026-05-04T23:49:55.489437419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:48538"
level=debug ts=2026-05-04T23:50:02.862535135Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:50:15.335271819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:52890"
ts=2026-05-04T23:50:19.457607268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:50:43.263184901Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:50:45.337022564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:34240"
level=debug ts=2026-05-04T23:50:46.09467274Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:50:49.459390342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:51:15.339509267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:47158"
ts=2026-05-04T23:51:19.461440445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:51:24.893581552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:58688"
level=debug ts=2026-05-04T23:51:35.818231944Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:51:35.819946983Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T23:51:43.262741826Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:51:49.463828868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:52:11.952403731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:52:19.465804392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-04T23:52:43.262554628Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:52:49.468139711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:52:54.898996722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:35842"
level=debug ts=2026-05-04T23:53:12.43729153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:53:19.470451654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:53:43.262312697Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:53:49.472103141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:53:55.501533202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:55294"
level=debug ts=2026-05-04T23:54:01.776126698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:54:19.473839056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:54:24.903850046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:56004"
level=debug ts=2026-05-04T23:54:42.647351756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T23:54:43.262382877Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:54:49.475557715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:55:15.351335385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49736"
ts=2026-05-04T23:55:19.477765556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:55:25.506361396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:45568"
level=debug ts=2026-05-04T23:55:30.681565921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:55:30.683018462Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T23:55:43.262411429Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:55:49.480303407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:55:54.908115742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:42162"
ts=2026-05-04T23:55:55.508118294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:46052"
ts=2026-05-04T23:56:15.354937497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:52462"
level=debug ts=2026-05-04T23:56:15.822297742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:56:19.481804428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:56:24.910335503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:48126"
level=debug ts=2026-05-04T23:56:43.26298987Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:56:49.483359978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:56:54.911870871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:42902"
level=debug ts=2026-05-04T23:57:16.375600493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:57:19.485522642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:57:25.51324748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:35348"
level=debug ts=2026-05-04T23:57:43.262934455Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:57:45.359370743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:37236"
ts=2026-05-04T23:57:49.487851502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:57:54.915390942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:45756"
ts=2026-05-04T23:57:55.514877387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:49904"
level=debug ts=2026-05-04T23:58:06.834551957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:58:19.490215819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:58:25.516521064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:42670"
level=debug ts=2026-05-04T23:58:43.262616504Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:58:45.363633544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48882"
level=debug ts=2026-05-04T23:58:45.445475594Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:58:49.49238763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-04T23:58:55.518835768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:40314"
ts=2026-05-04T23:59:19.494350065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:59:24.920504455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:56948"
level=debug ts=2026-05-04T23:59:31.677965749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:59:31.67965457Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T23:59:43.263006896Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-04T23:59:49.496756306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:59:54.923017252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53738"
level=debug ts=2026-05-05T00:00:13.145006812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:00:15.36998219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:58868"
ts=2026-05-05T00:00:19.499031123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-05T00:00:24.924493713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:35806"
level=debug ts=2026-05-05T00:00:43.263066563Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:00:49.50165873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-05T00:01:03.48629732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:01:15.372987127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:46314"
ts=2026-05-05T00:01:19.504305766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-05T00:01:43.263183544Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:01:45.374776446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:58238"
ts=2026-05-05T00:01:49.506991019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-05T00:01:54.429162131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:01:54.929578665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:40432"
ts=2026-05-05T00:01:55.528678247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:40520"
ts=2026-05-05T00:02:19.509283733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:02:43.262739286Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-05-05T00:02:47.390215634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:02:49.511326046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:02:54.933232549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:41664"
ts=2026-05-05T00:03:19.51321976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:03:31.468960033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:03:31.47046025Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T00:03:43.262867257Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:03:49.515219099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:04:11.289076883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:04:19.516787315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:04:43.26269688Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:04:49.518426775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:04:54.93953602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:52752"
level=debug ts=2026-05-05T00:04:59.494490942Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:05:19.520446268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:05:25.538947041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:35858"
level=debug ts=2026-05-05T00:05:43.263237898Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:05:49.522801707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-05T00:05:57.882400082Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:06:15.387441786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:32974"
ts=2026-05-05T00:06:19.525129734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:06:36.478610466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T00:06:43.262280445Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:06:45.390115676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:43640"
ts=2026-05-05T00:06:49.526629037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:06:55.544410266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:50694"
ts=2026-05-05T00:07:19.528952173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-05T00:07:30.026148581Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:07:30.027737643Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T00:07:43.262571997Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:07:49.531414856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:07:55.548447531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:59716"
level=debug ts=2026-05-05T00:08:06.683456313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:08:19.533263894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-05T00:08:43.262221627Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:08:49.535483657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-05T00:08:50.543601567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:08:55.551950996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:46940"
ts=2026-05-05T00:09:15.399006002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:57678"
ts=2026-05-05T00:09:19.537031869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:09:25.5545174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:55080"
level=debug ts=2026-05-05T00:09:33.70798696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T00:09:43.262991904Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:09:49.539132222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-05T00:09:55.55637418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:44874"
level=debug ts=2026-05-05T00:10:15.104396311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:10:19.540927524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-05T00:10:43.262853688Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:10:49.542891658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:10:54.956727212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:34468"
level=debug ts=2026-05-05T00:11:05.232291949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:11:05.233855364Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:11:19.544634361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:11:25.562329623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:47418"
level=debug ts=2026-05-05T00:11:43.263432025Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:11:45.406933802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48770"
ts=2026-05-05T00:11:49.546319116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-05T00:12:05.34164956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:12:19.548377863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:12:24.962078183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:34604"
ts=2026-05-05T00:12:25.566086198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:50168"
level=debug ts=2026-05-05T00:12:43.262890458Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:12:49.550885605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-05T00:12:55.567843607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:53312"
level=debug ts=2026-05-05T00:13:05.382331964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:13:19.553166074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-05T00:13:24.965966938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46806"
ts=2026-05-05T00:13:25.569834356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:54878"
level=debug ts=2026-05-05T00:13:40.235952671Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T00:13:43.262738518Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:13:45.415753473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60640"
ts=2026-05-05T00:13:49.555877263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:14:19.557475286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:14:26.445489467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T00:14:43.263008482Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:14:49.559862583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:15:19.562021562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:15:22.140321968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:15:22.141773847Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T00:15:43.262621474Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:15:49.563633094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:15:54.973412977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:57808"
ts=2026-05-05T00:16:19.566000837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-05T00:16:20.826127048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:16:24.975191297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:37742"
level=debug ts=2026-05-05T00:16:43.262498428Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:16:49.568176712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-05T00:16:55.582036608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:33128"
level=debug ts=2026-05-05T00:17:10.19207917Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:17:19.569778598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-05T00:17:24.979248983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55968"
level=debug ts=2026-05-05T00:17:43.262370118Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:17:49.571878421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-05T00:18:04.428368029Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:18:19.574357425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:18:24.982721602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:60880"
level=debug ts=2026-05-05T00:18:43.262340775Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:18:45.431648159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:59676"
ts=2026-05-05T00:18:49.576155409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:18:54.984295318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:42938"
level=debug ts=2026-05-05T00:19:01.818673866Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:19:19.577938716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:19:43.262306469Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-05-05T00:19:47.517092496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:19:47.518666191Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:19:49.580279775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-05T00:20:19.582739879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:20:43.262748992Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-05-05T00:20:45.101896737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:20:45.437560207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:57448"
ts=2026-05-05T00:20:49.58446623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:21:19.586391662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:21:33.799045569Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T00:21:43.263005651Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:21:49.588652464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:22:15.442318254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:54222"
ts=2026-05-05T00:22:19.59035352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:22:26.859711386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T00:22:43.26207724Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:22:49.591930895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-05T00:22:54.994401844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55140"
ts=2026-05-05T00:23:15.444977887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:54380"
ts=2026-05-05T00:23:19.593641281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-05T00:23:23.764943691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T00:23:43.262416249Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:23:49.596006351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
level=debug ts=2026-05-05T00:23:59.109786044Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:23:59.11139355Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:24:19.598386397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:24:25.605500598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:53938"
level=debug ts=2026-05-05T00:24:43.262323752Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:24:45.4494949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:42658"
level=debug ts=2026-05-05T00:24:46.485497893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:24:49.600123932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-b077c5d9 10.130.0.65:7946"
ts=2026-05-05T00:25:19.601764282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:25:25.609064889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:36466"
level=debug ts=2026-05-05T00:25:33.102612262Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T00:25:43.26331676Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
ts=2026-05-05T00:25:49.603298678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:25:55.003576117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:43238"
level=debug ts=2026-05-05T00:26:08.064369625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:26:19.605973478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"