level=info ts=2026-04-23T08:09:19.162878006Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-23T08:09:19.16303539Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-23T08:09:19.16442051Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-23T08:09:19.164721879Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-23T08:09:19.16479473Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.88 inf=eth0
level=debug ts=2026-04-23T08:09:19.165213347Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-23T08:09:19.165305489Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.88 inf=eth0
level=info ts=2026-04-23T08:09:19.165320489Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-7e16cc4f
ts=2026-04-23T08:09:19.165405933Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-23T08:09:19.165424276Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.88 advertisePort=7946
level=debug ts=2026-04-23T08:09:19.165449707Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.88 advertisePort=7946
level=info ts=2026-04-23T08:09:19.165555899Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-23T08:09:19.178715703Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-23T08:09:19.178741815Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-23T08:09:19.178757598Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=13.203995ms
level=error ts=2026-04-23T08:09:19.178768213Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-23T08:09:19.178778646Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-04-23T08:09:19.180738757Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-23T08:09:19.180772535Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-23T08:09:19.180774792Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-23T08:09:19.180853136Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-23T08:09:19.180877459Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-04-23T08:09:19.180890954Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-23T08:09:19.180975079Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-23T08:09:19.181000178Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-23T08:09:19.181204136Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-23T08:09:19.181233568Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-23T08:09:19.181302545Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-23T08:09:19.181294923Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-23T08:09:19.181323067Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-23T08:09:19.181527706Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-23T08:09:19.181565124Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-23T08:09:19.18162284Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-04-23T08:09:19.181832315Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-23T08:09:19.181847771Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-23T08:09:20.181971188Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-23T08:09:20.376635712Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-23T08:09:20.376689463Z caller=loki.go:599 msg="Loki started" startup_time=1.274572844s
ts=2026-04-23T08:09:20.817631041Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-23T08:09:20.817660919Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T08:09:24.640369036Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-23T08:09:24.640399719Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-23T08:09:25.377415845Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-23T08:09:25.377475772Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-23T08:09:30.570058602Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-23T08:09:30.57008253Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-23T08:09:34.747907356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T08:09:39.213889398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:39910"
ts=2026-04-23T08:09:41.639114179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.73:7946"
ts=2026-04-23T08:09:41.641996272Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.72:7946: dial tcp 10.128.0.72:7946: connect: connection refused"
ts=2026-04-23T08:09:41.642144555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.88:7946"
ts=2026-04-23T08:09:41.642160126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:36272"
ts=2026-04-23T08:09:41.643745371Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.87:7946: dial tcp 10.129.0.87:7946: connect: connection refused"
level=info ts=2026-04-23T08:09:41.643760348Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=22.464975522s
ts=2026-04-23T08:09:42.815716647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:41940"
ts=2026-04-23T08:09:46.025171208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:39396"
level=debug ts=2026-04-23T08:10:00.513023646Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T08:10:12.624558676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
level=debug ts=2026-04-23T08:10:18.169661759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T08:10:42.626956301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
level=debug ts=2026-04-23T08:10:42.831629085Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T08:10:48.148658299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:36036"
ts=2026-04-23T08:11:01.748129759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:38902"
ts=2026-04-23T08:11:12.629499122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T08:11:16.096901022Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-23T08:11:16.098784482Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T08:11:42.631963723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
level=debug ts=2026-04-23T08:12:04.860385497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T08:12:12.634099951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:12:42.636445814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
ts=2026-04-23T08:12:47.840374317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:36132"
ts=2026-04-23T08:13:12.638193517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
level=debug ts=2026-04-23T08:13:14.311482099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T08:13:42.640147799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:13:47.845192979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:41972"
ts=2026-04-23T08:14:12.642959942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T08:14:16.69350898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T08:14:18.164557102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:44224"
ts=2026-04-23T08:14:42.645731005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
level=debug ts=2026-04-23T08:15:06.059695416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T08:15:12.648378567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:15:42.650959394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
ts=2026-04-23T08:15:48.171924457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:49672"
level=debug ts=2026-04-23T08:16:08.595008651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-23T08:16:08.596625465Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T08:16:12.653235277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:16:18.174379224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:54354"
ts=2026-04-23T08:16:42.655094463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
ts=2026-04-23T08:16:48.177769163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:51958"
level=debug ts=2026-04-23T08:17:10.841214609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:17:12.657251752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:17:17.863422843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:41484"
ts=2026-04-23T08:17:18.180975755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46022"
ts=2026-04-23T08:17:42.660250627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T08:17:52.873017093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:18:12.661722241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
ts=2026-04-23T08:18:18.18687922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53430"
level=debug ts=2026-04-23T08:18:29.745750641Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:18:31.790127657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:57094"
ts=2026-04-23T08:18:42.66375803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:18:47.870535327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:54518"
ts=2026-04-23T08:18:48.189561706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:42232"
ts=2026-04-23T08:19:12.666088721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
ts=2026-04-23T08:19:17.873806369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:36634"
level=info ts=2026-04-23T08:19:25.377710018Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-23T08:19:25.381454754Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T08:19:25.383363Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T08:19:25.385033882Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T08:19:25.385069709Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-23T08:19:25.385081354Z caller=tables_manager.go:145 msg="compactor started"
level=debug ts=2026-04-23T08:19:31.68572825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:19:31.794535093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:35248"
ts=2026-04-23T08:19:42.668391136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:19:47.877211962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:52508"
ts=2026-04-23T08:20:01.795433515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:37638"
ts=2026-04-23T08:20:12.671299718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T08:20:28.233466182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T08:20:28.235351361Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T08:20:31.796764516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:49008"
ts=2026-04-23T08:20:42.673796234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:21:12.676365189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:21:17.884279884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:59614"
level=debug ts=2026-04-23T08:21:26.956833358Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:21:31.800761087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:37664"
ts=2026-04-23T08:21:42.679136546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:21:48.204803282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:52716"
level=debug ts=2026-04-23T08:22:01.641342133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:22:12.681453402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
ts=2026-04-23T08:22:18.207214281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:54162"
level=debug ts=2026-04-23T08:22:42.482774338Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:22:42.683559723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:22:47.891105278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:54134"
ts=2026-04-23T08:23:12.686032377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
ts=2026-04-23T08:23:31.811736985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:32976"
level=debug ts=2026-04-23T08:23:37.944762361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:23:42.68785872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:23:48.214535293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53996"
ts=2026-04-23T08:24:01.813231022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:57826"
ts=2026-04-23T08:24:12.691243735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T08:24:12.846598834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T08:24:12.848391128Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T08:24:42.694296071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:24:47.901275787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:50200"
ts=2026-04-23T08:24:48.22086713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:41938"
level=debug ts=2026-04-23T08:24:50.074753963Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:25:12.696582513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T08:25:24.159395687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:25:42.699029736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
ts=2026-04-23T08:25:47.906867733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:58760"
ts=2026-04-23T08:25:48.226124599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:47918"
ts=2026-04-23T08:26:01.823202236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:53304"
ts=2026-04-23T08:26:12.701187822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
level=debug ts=2026-04-23T08:26:15.095290387Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:26:42.704320994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:26:47.911995075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:56752"
level=debug ts=2026-04-23T08:26:49.47624857Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:27:01.828209274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:43450"
ts=2026-04-23T08:27:12.707859496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:27:18.235099416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:45980"
level=debug ts=2026-04-23T08:27:37.958643699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T08:27:37.960772964Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T08:27:42.710356898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
ts=2026-04-23T08:28:12.712910474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:28:18.239799063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:38058"
level=debug ts=2026-04-23T08:28:21.463403804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:28:31.83638607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:58448"
ts=2026-04-23T08:28:42.714999458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
ts=2026-04-23T08:29:12.716264188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
level=debug ts=2026-04-23T08:29:19.760782719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T08:29:25.395261305Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T08:29:25.39714565Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T08:29:25.399171753Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T08:29:25.399199624Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T08:29:31.842084701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:42784"
ts=2026-04-23T08:29:42.717329214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
ts=2026-04-23T08:30:01.843019931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:51248"
level=debug ts=2026-04-23T08:30:11.670225041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:30:12.720083551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:30:17.931555952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:57314"
ts=2026-04-23T08:30:42.723502635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
level=debug ts=2026-04-23T08:30:55.828964911Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:31:01.847663832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:46454"
ts=2026-04-23T08:31:12.726982165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:31:17.938037517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:38416"
level=debug ts=2026-04-23T08:31:37.090562136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T08:31:37.092439619Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T08:31:42.730437168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:32:12.73408885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
level=debug ts=2026-04-23T08:32:38.821306111Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:32:42.736835432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:32:48.265983912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59774"
ts=2026-04-23T08:33:12.738580547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
level=debug ts=2026-04-23T08:33:14.491140289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:33:42.740795582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:33:48.273785215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:52402"
level=debug ts=2026-04-23T08:34:12.308258009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:34:12.743361845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:34:42.746529232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:35:01.873167281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:54930"
level=debug ts=2026-04-23T08:35:10.253773995Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:35:12.749963541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:35:17.959508972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:35540"
ts=2026-04-23T08:35:42.751944164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
level=debug ts=2026-04-23T08:36:04.5084749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T08:36:04.5102926Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T08:36:12.753111918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
ts=2026-04-23T08:36:17.971580641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:59008"
ts=2026-04-23T08:36:42.754492626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
level=debug ts=2026-04-23T08:36:44.641687761Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:37:12.755345171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
level=debug ts=2026-04-23T08:37:38.877967091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:37:42.756886313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
ts=2026-04-23T08:37:47.980997151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:51160"
ts=2026-04-23T08:37:48.292257527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:54496"
ts=2026-04-23T08:38:12.758002378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cf75220c 10.129.0.89:7946"
ts=2026-04-23T08:38:31.893806349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:40304"
level=debug ts=2026-04-23T08:38:38.371396227Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:38:42.760030416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:38:47.987897583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:46280"
ts=2026-04-23T08:38:48.297151302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55074"
ts=2026-04-23T08:39:04.584585521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:37292"
ts=2026-04-23T08:39:04.596124901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:37294"
ts=2026-04-23T08:39:06.167359657Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-cf75220c (timeout reached)"
ts=2026-04-23T08:39:09.166654109Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-cf75220c has failed, no acks received"
ts=2026-04-23T08:39:12.762160476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-62811d50 10.129.0.89:7946"
level=error ts=2026-04-23T08:39:25.402232247Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T08:39:25.404213859Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T08:39:25.405893842Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T08:39:25.405921906Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T08:39:29.167479777Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-cf75220c as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-23T08:39:32.41297847Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:39:38.232140759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:40530"
ts=2026-04-23T08:39:42.764353769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:39:48.301375976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:51970"
ts=2026-04-23T08:40:08.233735312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:50512"
ts=2026-04-23T08:40:12.767319514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T08:40:32.450497765Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T08:40:32.45259836Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T08:40:38.235287617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:50204"
ts=2026-04-23T08:40:42.770788989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:41:12.772806264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-62811d50 10.129.0.89:7946"
ts=2026-04-23T08:41:18.309740293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:33894"
level=debug ts=2026-04-23T08:41:19.772206784Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:41:42.775352652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:42:08.24213312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:32918"
level=debug ts=2026-04-23T08:42:10.284030369Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:42:12.777729236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-62811d50 10.129.0.89:7946"
ts=2026-04-23T08:42:18.314313228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46126"
ts=2026-04-23T08:42:38.243712999Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:36426"
ts=2026-04-23T08:42:42.780017641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:42:48.008239182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:57622"
level=debug ts=2026-04-23T08:43:07.259266175Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:43:12.783027691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:43:38.247216073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:59002"
ts=2026-04-23T08:43:42.78523597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-62811d50 10.129.0.89:7946"
ts=2026-04-23T08:43:48.322077485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:47520"
level=debug ts=2026-04-23T08:43:53.957782098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:44:12.78755859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:44:18.324966241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:33446"
level=debug ts=2026-04-23T08:44:32.169296375Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T08:44:32.170941402Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T08:44:42.790863757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:44:48.017047946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:46234"
ts=2026-04-23T08:44:48.327729795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:51144"
level=debug ts=2026-04-23T08:45:08.372157613Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:45:12.793068574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:45:42.795412473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:45:48.023528228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:55368"
ts=2026-04-23T08:45:48.333993981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:57618"
level=debug ts=2026-04-23T08:45:57.959288653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:46:12.798749172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:46:18.026591152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:49490"
ts=2026-04-23T08:46:18.336427052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53874"
level=debug ts=2026-04-23T08:46:41.638539715Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:46:42.801716966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:46:48.339018757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:35696"
ts=2026-04-23T08:47:12.804477797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:47:18.033546786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:36982"
level=debug ts=2026-04-23T08:47:36.196686707Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:47:42.806100321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-62811d50 10.129.0.89:7946"
ts=2026-04-23T08:47:48.343577373Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:44682"
level=debug ts=2026-04-23T08:48:11.457857051Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T08:48:11.459548709Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T08:48:12.80750214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-62811d50 10.129.0.89:7946"
ts=2026-04-23T08:48:18.039154994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:60884"
ts=2026-04-23T08:48:18.346937035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:49330"
ts=2026-04-23T08:48:38.275778201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:41046"
ts=2026-04-23T08:48:42.809739855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:48:46.166821952Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-62811d50 (timeout reached)"
ts=2026-04-23T08:48:48.04296626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:60074"
ts=2026-04-23T08:48:49.166269288Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-62811d50 has failed, no acks received"
level=debug ts=2026-04-23T08:48:50.489075598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:48:51.167062121Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-62811d50 (timeout reached)"
ts=2026-04-23T08:48:54.166450294Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-62811d50 has failed, no acks received"
ts=2026-04-23T08:48:56.89958951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:41826"
ts=2026-04-23T08:48:56.913180968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:41836"
ts=2026-04-23T08:49:09.16714524Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-62811d50 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T08:49:11.166034129Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-62811d50 (timeout reached)"
ts=2026-04-23T08:49:12.812022863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3d4c8bb5 10.129.0.89:7946"
ts=2026-04-23T08:49:14.166200267Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-62811d50 has failed, no acks received"
level=error ts=2026-04-23T08:49:25.389139212Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T08:49:25.390841771Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T08:49:25.392562911Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T08:49:25.392587916Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-23T08:49:34.137471228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:49:42.814427252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:49:48.353879613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:45862"
ts=2026-04-23T08:49:51.550119125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:41326"
ts=2026-04-23T08:50:12.817982324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:50:18.053693483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45054"
level=debug ts=2026-04-23T08:50:35.469967618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:50:42.81973059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3d4c8bb5 10.129.0.89:7946"
ts=2026-04-23T08:50:48.362760402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46860"
ts=2026-04-23T08:51:12.82212742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T08:51:14.312094724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:51:18.061016745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:48578"
ts=2026-04-23T08:51:21.560282113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:37550"
ts=2026-04-23T08:51:42.824522595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3d4c8bb5 10.129.0.89:7946"
level=debug ts=2026-04-23T08:51:55.179413512Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T08:51:55.181109152Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T08:52:12.827328807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:52:21.564138793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:40834"
ts=2026-04-23T08:52:42.829422872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3d4c8bb5 10.129.0.89:7946"
level=debug ts=2026-04-23T08:52:48.020964801Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:52:48.376174627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:48856"
ts=2026-04-23T08:53:12.830526524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3d4c8bb5 10.129.0.89:7946"
ts=2026-04-23T08:53:18.073590873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:59824"
ts=2026-04-23T08:53:42.832082184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3d4c8bb5 10.129.0.89:7946"
ts=2026-04-23T08:53:48.381968892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:47736"
level=debug ts=2026-04-23T08:53:48.95998287Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:53:51.571964066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:36262"
ts=2026-04-23T08:54:12.83311523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3d4c8bb5 10.129.0.89:7946"
ts=2026-04-23T08:54:18.079722545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:39436"
ts=2026-04-23T08:54:18.384862911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:52758"
level=debug ts=2026-04-23T08:54:31.078097421Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:54:42.834820798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:55:12.837314877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3d4c8bb5 10.129.0.89:7946"
level=debug ts=2026-04-23T08:55:15.522224737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:55:42.840173874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
level=debug ts=2026-04-23T08:55:59.074480594Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T08:55:59.076397884Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T08:56:12.842072255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3d4c8bb5 10.129.0.89:7946"
ts=2026-04-23T08:56:21.585900119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:36860"
level=debug ts=2026-04-23T08:56:35.34903166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:56:42.844710763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:56:51.586809012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:54030"
ts=2026-04-23T08:57:12.846634796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3d4c8bb5 10.129.0.89:7946"
level=debug ts=2026-04-23T08:57:24.862484467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:57:42.84937177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T08:57:48.401187288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:49598"
ts=2026-04-23T08:58:12.852457955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:58:18.404472443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:33940"
level=debug ts=2026-04-23T08:58:21.600636847Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T08:58:42.854690709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3d4c8bb5 10.129.0.89:7946"
ts=2026-04-23T08:58:48.40763391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:33348"
ts=2026-04-23T08:59:12.855788319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3d4c8bb5 10.129.0.89:7946"
level=debug ts=2026-04-23T08:59:13.507701534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T08:59:25.388695279Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T08:59:25.390379889Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T08:59:25.392138407Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T08:59:25.392167665Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T08:59:42.85791666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T08:59:48.111508591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:59932"
level=debug ts=2026-04-23T08:59:52.495844783Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T08:59:52.497474455Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T09:00:12.861080942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:00:18.415400333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:42956"
ts=2026-04-23T09:00:42.863499704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T09:00:46.559734018Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:00:48.118224004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:47076"
ts=2026-04-23T09:00:51.609344264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:40582"
ts=2026-04-23T09:01:12.865789624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3d4c8bb5 10.129.0.89:7946"
ts=2026-04-23T09:01:21.611067238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:39478"
ts=2026-04-23T09:01:25.62545279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:55022"
ts=2026-04-23T09:01:25.637196752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:55028"
level=debug ts=2026-04-23T09:01:25.807060006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:01:31.166616835Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-3d4c8bb5 (timeout reached)"
ts=2026-04-23T09:01:34.166408509Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-3d4c8bb5 has failed, no acks received"
ts=2026-04-23T09:01:41.166647824Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-3d4c8bb5 (timeout reached)"
ts=2026-04-23T09:01:42.867746952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:01:44.167119113Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-3d4c8bb5 has failed, no acks received"
ts=2026-04-23T09:01:54.166577346Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-3d4c8bb5 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T09:02:12.870983151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:02:18.425197736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:34260"
level=debug ts=2026-04-23T09:02:25.252888559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:02:42.87324118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:03:12.875829863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
level=debug ts=2026-04-23T09:03:18.530830742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:03:42.878240462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:03:48.133438409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:49426"
level=debug ts=2026-04-23T09:04:12.296026316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T09:04:12.297914204Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T09:04:12.880710336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:04:18.136745264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:58230"
ts=2026-04-23T09:04:22.291270481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:60866"
ts=2026-04-23T09:04:42.882925154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-61f12512 10.129.0.89:7946"
level=debug ts=2026-04-23T09:05:08.317416508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:05:12.885432152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:05:18.141717073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:52500"
ts=2026-04-23T09:05:42.888366154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T09:06:02.836005738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:06:12.891171444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:06:42.893643911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
level=debug ts=2026-04-23T09:06:59.064746755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:07:12.89676227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:07:18.44791615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:42756"
ts=2026-04-23T09:07:22.308298607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:49858"
ts=2026-04-23T09:07:42.89973942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:07:48.451561391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:43816"
ts=2026-04-23T09:07:52.309652247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:49282"
level=debug ts=2026-04-23T09:07:53.296449393Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:08:12.902478961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
level=debug ts=2026-04-23T09:08:30.456083996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T09:08:30.457684934Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T09:08:42.904860263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-61f12512 10.129.0.89:7946"
ts=2026-04-23T09:08:52.314907643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:59714"
ts=2026-04-23T09:09:12.906688223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:09:18.458236612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:49298"
ts=2026-04-23T09:09:22.316260814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:60846"
level=debug ts=2026-04-23T09:09:24.002907093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T09:09:25.387669982Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T09:09:25.389897304Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T09:09:25.39132011Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T09:09:25.391346135Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T09:09:42.908349557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-61f12512 10.129.0.89:7946"
ts=2026-04-23T09:10:12.910896232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:10:18.170730264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:57424"
ts=2026-04-23T09:10:18.463248997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:49206"
level=debug ts=2026-04-23T09:10:20.378016813Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:10:22.321415025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:46730"
ts=2026-04-23T09:10:42.915257904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:10:52.322574344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:52024"
level=debug ts=2026-04-23T09:11:09.072291102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:11:12.917714165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-61f12512 10.129.0.89:7946"
ts=2026-04-23T09:11:18.177581252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:39628"
ts=2026-04-23T09:11:18.469063041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59788"
ts=2026-04-23T09:11:42.918683445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-61f12512 10.129.0.89:7946"
ts=2026-04-23T09:11:48.180808698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:57484"
ts=2026-04-23T09:11:48.471639392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:48650"
level=debug ts=2026-04-23T09:12:03.996851305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:12:12.920863902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:12:18.474269726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:51354"
ts=2026-04-23T09:12:42.923728936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T09:12:46.318685129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T09:12:46.321111914Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T09:13:12.92622877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:13:18.188680041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:38132"
ts=2026-04-23T09:13:18.47875305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55850"
ts=2026-04-23T09:13:22.335832658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:56632"
level=debug ts=2026-04-23T09:13:30.508146603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:13:42.929477898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:13:48.481310821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:50130"
ts=2026-04-23T09:13:51.810261317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:39536"
ts=2026-04-23T09:13:51.821247987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:39542"
ts=2026-04-23T09:14:06.166517437Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-61f12512 (timeout reached)"
level=debug ts=2026-04-23T09:14:06.490493655Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:14:09.165722777Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-61f12512 has failed, no acks received"
ts=2026-04-23T09:14:12.932742468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:14:18.193881458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:59850"
ts=2026-04-23T09:14:42.936001652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
level=debug ts=2026-04-23T09:14:58.334899612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:14:59.915859836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:41524"
ts=2026-04-23T09:15:12.938702363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:15:18.199151692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:56118"
ts=2026-04-23T09:15:42.941760419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:15:48.202328327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:48720"
level=debug ts=2026-04-23T09:15:48.206826226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:16:12.94496955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:16:18.205393242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:49236"
level=debug ts=2026-04-23T09:16:40.241189471Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T09:16:40.24278942Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T09:16:42.946575349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-be531691 10.129.0.89:7946"
ts=2026-04-23T09:17:12.948885321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:17:29.928378986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:52460"
level=debug ts=2026-04-23T09:17:37.017559248Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:17:42.952034915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:18:12.954372829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-be531691 10.129.0.89:7946"
ts=2026-04-23T09:18:18.504326403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:52162"
level=debug ts=2026-04-23T09:18:38.532079027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:18:42.955496577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-be531691 10.129.0.89:7946"
ts=2026-04-23T09:18:48.507282464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:49886"
ts=2026-04-23T09:18:59.937327293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:59508"
ts=2026-04-23T09:19:12.957147256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=error ts=2026-04-23T09:19:25.388993026Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T09:19:25.390728429Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T09:19:25.39240354Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T09:19:25.392428407Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T09:19:29.939045388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:37020"
level=debug ts=2026-04-23T09:19:35.940281477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:19:42.959010392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-be531691 10.129.0.89:7946"
ts=2026-04-23T09:19:48.512215972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:45806"
ts=2026-04-23T09:19:59.94076685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:53562"
ts=2026-04-23T09:20:12.961679259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T09:20:15.85108118Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:20:18.514896962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:45426"
ts=2026-04-23T09:20:42.96444651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:20:59.945673288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:49588"
ts=2026-04-23T09:21:12.966888482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T09:21:16.156711115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T09:21:16.158338303Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T09:21:18.520944748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:47502"
ts=2026-04-23T09:21:42.969340884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-be531691 10.129.0.89:7946"
level=debug ts=2026-04-23T09:22:10.132960705Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:22:12.971803689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:22:18.239478766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:50048"
ts=2026-04-23T09:22:18.525714747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:40916"
ts=2026-04-23T09:22:42.973846014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-be531691 10.129.0.89:7946"
ts=2026-04-23T09:22:48.242146359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:34646"
level=debug ts=2026-04-23T09:22:55.550563347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:23:12.976244674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T09:23:39.485741806Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:23:42.979472274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:24:12.981945591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-be531691 10.129.0.89:7946"
level=debug ts=2026-04-23T09:24:21.368075664Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:24:42.984141076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:24:59.969090805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:54576"
level=debug ts=2026-04-23T09:25:11.999798676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T09:25:12.001387696Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T09:25:12.986600161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:25:42.989675711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
level=debug ts=2026-04-23T09:25:52.057129078Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:25:55.874010995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:34776"
ts=2026-04-23T09:25:55.885183362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:34786"
ts=2026-04-23T09:26:06.166746941Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-be531691 (timeout reached)"
ts=2026-04-23T09:26:09.166950027Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-be531691 has failed, no acks received"
ts=2026-04-23T09:26:12.993132175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:26:18.545296852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:58148"
ts=2026-04-23T09:26:30.853529565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:38382"
ts=2026-04-23T09:26:42.996840364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
level=debug ts=2026-04-23T09:26:45.223516786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:26:48.547902269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46520"
ts=2026-04-23T09:27:00.855332076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:50502"
ts=2026-04-23T09:27:12.999930876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:27:18.550756445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53966"
level=debug ts=2026-04-23T09:27:34.385274146Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:27:43.00191221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4b2767a4 10.129.0.89:7946"
ts=2026-04-23T09:27:48.553503801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59878"
level=debug ts=2026-04-23T09:28:09.074875112Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:28:13.004643893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:28:43.007863358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:28:48.273099468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45340"
level=debug ts=2026-04-23T09:29:01.095626703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T09:29:01.097272378Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T09:29:13.010516536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:29:18.562815677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:49258"
level=error ts=2026-04-23T09:29:25.389527531Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T09:29:25.391265819Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T09:29:25.393054152Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T09:29:25.393081437Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-23T09:29:36.884964147Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:29:43.014185265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:30:13.016930383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:30:18.279489042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:39500"
ts=2026-04-23T09:30:18.568564216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:40126"
ts=2026-04-23T09:30:30.873970592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:34470"
level=debug ts=2026-04-23T09:30:31.741721255Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:30:43.0198636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:30:48.571093317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:60396"
ts=2026-04-23T09:31:00.87552171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:49738"
ts=2026-04-23T09:31:13.022207746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T09:31:14.527286166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:31:18.573850472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:51926"
ts=2026-04-23T09:31:43.023734978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4b2767a4 10.129.0.89:7946"
ts=2026-04-23T09:31:48.289820321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:41776"
level=debug ts=2026-04-23T09:32:01.684148559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:32:13.02494126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4b2767a4 10.129.0.89:7946"
ts=2026-04-23T09:32:18.579308519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:36404"
level=debug ts=2026-04-23T09:32:36.389812806Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T09:32:36.391427746Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T09:32:43.032758774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:33:13.037777874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T09:33:17.663238985Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:33:30.889445134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:46978"
ts=2026-04-23T09:33:43.04036259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:33:48.586150475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:44128"
ts=2026-04-23T09:33:56.165966866Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-4b2767a4 (timeout reached)"
level=debug ts=2026-04-23T09:33:57.119136519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:33:59.166173941Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-4b2767a4 has failed, no acks received"
ts=2026-04-23T09:34:06.930795132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:59198"
ts=2026-04-23T09:34:06.94274169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:59212"
ts=2026-04-23T09:34:11.166428746Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-4b2767a4 (timeout reached)"
ts=2026-04-23T09:34:13.042983739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:34:14.166066147Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-4b2767a4 has failed, no acks received"
ts=2026-04-23T09:34:18.590713389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:45028"
ts=2026-04-23T09:34:19.166281279Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-4b2767a4 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T09:34:21.166497699Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-4b2767a4 (timeout reached)"
ts=2026-04-23T09:34:24.166462743Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-4b2767a4 has failed, no acks received"
level=debug ts=2026-04-23T09:34:42.913812544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:34:43.044512989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4fcc0c35 10.129.0.89:7946"
ts=2026-04-23T09:35:13.047138875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
level=debug ts=2026-04-23T09:35:22.969227961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:35:43.050778074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
level=debug ts=2026-04-23T09:36:12.350870927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T09:36:12.353262857Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T09:36:13.054273902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:36:18.600489342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:34508"
ts=2026-04-23T09:36:43.057096616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:36:48.31796944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:36036"
ts=2026-04-23T09:36:48.60302554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59808"
level=debug ts=2026-04-23T09:36:56.167269441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:37:13.059891634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:37:43.062545804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:37:48.322878325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:55034"
level=debug ts=2026-04-23T09:37:50.52695804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:38:13.064692058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4fcc0c35 10.129.0.89:7946"
level=debug ts=2026-04-23T09:38:33.920376521Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:38:36.119658796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:51304"
ts=2026-04-23T09:38:43.066183894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4fcc0c35 10.129.0.89:7946"
ts=2026-04-23T09:38:48.61412508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:37698"
ts=2026-04-23T09:39:13.068017404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T09:39:16.836836258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:39:18.330766879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:58456"
ts=2026-04-23T09:39:18.616847874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:50208"
level=error ts=2026-04-23T09:39:25.387890444Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T09:39:25.38949317Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T09:39:25.391444132Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T09:39:25.391467928Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T09:39:36.124147364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:34810"
ts=2026-04-23T09:39:43.071013543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:39:48.619358758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46426"
level=debug ts=2026-04-23T09:39:52.581164861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T09:39:52.582980616Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T09:40:13.073804354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:40:18.622594292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53538"
ts=2026-04-23T09:40:43.076493652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
level=debug ts=2026-04-23T09:40:45.231811093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:40:48.337765952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:40304"
ts=2026-04-23T09:41:06.130862846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:57076"
ts=2026-04-23T09:41:13.078954572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
level=debug ts=2026-04-23T09:41:38.132745751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:41:43.081652965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:42:13.083758715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4fcc0c35 10.129.0.89:7946"
ts=2026-04-23T09:42:18.344772738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:43318"
level=debug ts=2026-04-23T09:42:40.021241956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:42:43.084945335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4fcc0c35 10.129.0.89:7946"
ts=2026-04-23T09:42:48.633267875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:52792"
ts=2026-04-23T09:43:01.166271904Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-4fcc0c35 (timeout reached)"
ts=2026-04-23T09:43:04.166934489Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-4fcc0c35 has failed, no acks received"
ts=2026-04-23T09:43:13.0874391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:43:16.167016545Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-4fcc0c35 (timeout reached)"
level=debug ts=2026-04-23T09:43:16.903386117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:43:18.350729757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:56476"
ts=2026-04-23T09:43:18.635921529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46014"
ts=2026-04-23T09:43:19.167006104Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-4fcc0c35 has failed, no acks received"
ts=2026-04-23T09:43:20.852329401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:44950"
ts=2026-04-23T09:43:20.864047975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:44952"
ts=2026-04-23T09:43:21.167219086Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-4fcc0c35 (timeout reached)"
ts=2026-04-23T09:43:24.167995411Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-4fcc0c35 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T09:43:24.168044294Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-4fcc0c35 has failed, no acks received"
ts=2026-04-23T09:43:43.089070309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-05c29a36 10.129.0.89:7946"
level=debug ts=2026-04-23T09:43:55.360482808Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T09:43:55.362296129Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T09:44:13.091405781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:44:27.033206129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:36766"
level=debug ts=2026-04-23T09:44:30.637004523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:44:43.094619472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:44:57.034395234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:35988"
ts=2026-04-23T09:45:13.098194724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
level=debug ts=2026-04-23T09:45:14.365033393Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:45:43.101781556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:45:48.648238201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:52696"
level=debug ts=2026-04-23T09:45:56.103715729Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:45:57.03888578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:34878"
ts=2026-04-23T09:46:13.10426793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-05c29a36 10.129.0.89:7946"
ts=2026-04-23T09:46:27.040270199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:54538"
ts=2026-04-23T09:46:43.105476163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-05c29a36 10.129.0.89:7946"
level=debug ts=2026-04-23T09:46:46.766250097Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:46:48.37144725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:44996"
ts=2026-04-23T09:47:13.107811264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:47:18.374840092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:33358"
level=debug ts=2026-04-23T09:47:25.775942278Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T09:47:25.777505624Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T09:47:27.044463571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:45384"
ts=2026-04-23T09:47:43.110844679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:48:13.112575432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-05c29a36 10.129.0.89:7946"
level=debug ts=2026-04-23T09:48:20.828141324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:48:27.047938479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:36498"
ts=2026-04-23T09:48:43.114085764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-05c29a36 10.129.0.89:7946"
level=debug ts=2026-04-23T09:49:09.432541708Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:49:13.116959548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
level=error ts=2026-04-23T09:49:25.389235668Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T09:49:25.390985913Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T09:49:25.392598734Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T09:49:25.392638375Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T09:49:36.167429153Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-05c29a36 (timeout reached)"
ts=2026-04-23T09:49:39.16728365Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-05c29a36 has failed, no acks received"
ts=2026-04-23T09:49:43.120105989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6ndm6-c2d43030 10.128.0.73:7946"
ts=2026-04-23T09:49:51.166272511Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-05c29a36 (timeout reached)"
ts=2026-04-23T09:49:54.166360201Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-05c29a36 has failed, no acks received"
ts=2026-04-23T09:49:54.294875014Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-05c29a36 as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-23T09:50:00.424996542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T09:50:13.123107962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-qgnbw-98d006c2 10.130.0.65:7946"
ts=2026-04-23T09:50:14.846536212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:45266"
ts=2026-04-23T09:50:14.858055725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:45268"