level=info ts=2026-04-20T20:21:47.00566565Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-20T20:21:47.005695393Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-20T20:21:47.006554168Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-20T20:21:47.006882953Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T20:21:47.006964478Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.84 inf=eth0
level=debug ts=2026-04-20T20:21:47.007946754Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T20:21:47.008013152Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.84 inf=eth0
level=info ts=2026-04-20T20:21:47.00811345Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-bdcbebaa
ts=2026-04-20T20:21:47.00821275Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-20T20:21:47.008229275Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.84 advertisePort=7946
level=debug ts=2026-04-20T20:21:47.008251241Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.84 advertisePort=7946
level=info ts=2026-04-20T20:21:47.008392022Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-20T20:21:47.013953821Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T20:21:47.013972445Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T20:21:47.013985609Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.595169ms
level=error ts=2026-04-20T20:21:47.013994574Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-20T20:21:47.014008652Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-20T20:21:47.016550785Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T20:21:47.016566068Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T20:21:47.02190071Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-20T20:21:47.021913546Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-20T20:21:47.021935431Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-20T20:21:47.021930352Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-20T20:21:47.021960539Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-20T20:21:47.021981159Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-20T20:21:47.022051687Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-20T20:21:47.022067643Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-20T20:21:47.022224643Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-20T20:21:47.022232187Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-20T20:21:47.022522847Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-20T20:21:47.022298367Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-20T20:21:47.02253146Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-20T20:21:47.022699866Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-20T20:21:47.202641746Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-20T20:21:47.202696875Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-20T20:21:48.20337409Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-20T20:21:48.387898104Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-20T20:21:48.387948881Z caller=loki.go:599 msg="Loki started" startup_time=1.447272119s
ts=2026-04-20T20:21:48.816695647Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T20:21:48.816724371Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T20:21:52.490147373Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T20:21:52.490175095Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T20:21:53.388418808Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-20T20:21:53.388468927Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-20T20:21:58.472634746Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T20:21:58.472659104Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T20:22:02.911921454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T20:22:04.060248514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:32976"
ts=2026-04-20T20:22:04.318383759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:49076"
ts=2026-04-20T20:22:06.262907159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:49814"
ts=2026-04-20T20:22:11.255360101Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.70:7946: dial tcp 10.128.0.70:7946: connect: connection refused"
ts=2026-04-20T20:22:11.255465785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.84:7946"
ts=2026-04-20T20:22:11.255492574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:39308"
level=info ts=2026-04-20T20:22:11.255848267Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=24.241833299s
level=debug ts=2026-04-20T20:22:21.445708814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T20:22:30.954187881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:33762"
ts=2026-04-20T20:22:35.816832309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
level=debug ts=2026-04-20T20:22:41.257532995Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T20:22:43.037430723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:40004"
ts=2026-04-20T20:23:00.957212981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56976"
level=debug ts=2026-04-20T20:23:05.706845663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T20:23:05.818408924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
ts=2026-04-20T20:23:30.689329491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:42226"
ts=2026-04-20T20:23:30.959051154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58800"
ts=2026-04-20T20:23:35.819880836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
level=debug ts=2026-04-20T20:23:38.452538844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T20:23:38.454104769Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T20:24:05.821746062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
level=debug ts=2026-04-20T20:24:17.015829571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T20:24:30.693675981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:58490"
ts=2026-04-20T20:24:35.823235816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
ts=2026-04-20T20:24:43.044959677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:55662"
ts=2026-04-20T20:25:00.965420051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:59152"
ts=2026-04-20T20:25:05.825186508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
level=debug ts=2026-04-20T20:25:11.034452144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T20:25:30.698090975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:46610"
ts=2026-04-20T20:25:30.968235308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34700"
ts=2026-04-20T20:25:35.827163122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
ts=2026-04-20T20:25:43.048640841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:60258"
ts=2026-04-20T20:26:00.700250079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:44792"
ts=2026-04-20T20:26:05.829143338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
level=debug ts=2026-04-20T20:26:09.771144642Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T20:26:35.830703474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
ts=2026-04-20T20:26:43.051930873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:37978"
ts=2026-04-20T20:27:00.704525485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:52968"
ts=2026-04-20T20:27:00.973819709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:33540"
ts=2026-04-20T20:27:05.831888108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
level=debug ts=2026-04-20T20:27:11.407823001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T20:27:13.053003154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:39458"
ts=2026-04-20T20:27:35.833832073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
ts=2026-04-20T20:28:00.978035555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56344"
ts=2026-04-20T20:28:05.835844943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
level=debug ts=2026-04-20T20:28:21.367145116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T20:28:21.368944031Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T20:28:30.709629586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:42396"
ts=2026-04-20T20:28:35.837789198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
level=debug ts=2026-04-20T20:28:56.317608908Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:29:00.711787731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:58720"
ts=2026-04-20T20:29:05.839705066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
ts=2026-04-20T20:29:13.061244987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:49650"
ts=2026-04-20T20:29:35.842118579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
level=debug ts=2026-04-20T20:29:42.818670203Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:30:05.844044283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
ts=2026-04-20T20:30:13.06415815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:48268"
level=debug ts=2026-04-20T20:30:30.368121731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:30:30.718230594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:54910"
ts=2026-04-20T20:30:30.986159645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:43770"
ts=2026-04-20T20:30:35.846343352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
ts=2026-04-20T20:31:05.848570199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
ts=2026-04-20T20:31:13.067023214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:43358"
level=debug ts=2026-04-20T20:31:29.45497424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:31:30.722065776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:51764"
ts=2026-04-20T20:31:35.850217792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
level=info ts=2026-04-20T20:31:53.388789249Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-20T20:31:53.39270376Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T20:31:53.394832198Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T20:31:53.39656101Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T20:31:53.396587024Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T20:31:53.396598848Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-20T20:32:00.991831946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:46870"
ts=2026-04-20T20:32:05.851650966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
level=debug ts=2026-04-20T20:32:11.832169638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T20:32:11.833682582Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T20:32:13.070898996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:33936"
ts=2026-04-20T20:32:30.994178552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58794"
ts=2026-04-20T20:32:35.852897315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
level=debug ts=2026-04-20T20:33:04.642741806Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:33:05.854593965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
ts=2026-04-20T20:33:35.856735543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
level=debug ts=2026-04-20T20:33:52.190608361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:34:00.999364672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:54730"
ts=2026-04-20T20:34:05.858371315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
ts=2026-04-20T20:34:13.079064519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:60136"
ts=2026-04-20T20:34:31.001533883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:44106"
ts=2026-04-20T20:34:35.860617778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
ts=2026-04-20T20:34:43.079798201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:45332"
level=debug ts=2026-04-20T20:34:45.441415749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:35:05.862792392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
ts=2026-04-20T20:35:30.73664968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:45782"
ts=2026-04-20T20:35:35.865195276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
ts=2026-04-20T20:35:43.08343647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:33482"
level=debug ts=2026-04-20T20:35:44.526662769Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:36:01.007363653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:33440"
ts=2026-04-20T20:36:05.867252422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
level=debug ts=2026-04-20T20:36:25.278452553Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T20:36:25.280375957Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T20:36:35.868692291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
ts=2026-04-20T20:37:00.742301237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:58980"
ts=2026-04-20T20:37:01.011011196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:52222"
level=debug ts=2026-04-20T20:37:05.15567415Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:37:05.870717145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
ts=2026-04-20T20:37:13.088300476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:48936"
ts=2026-04-20T20:37:30.744883547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:40966"
ts=2026-04-20T20:37:35.872695364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
level=debug ts=2026-04-20T20:37:52.932536962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:38:00.746946038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:59302"
ts=2026-04-20T20:38:05.874841597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
ts=2026-04-20T20:38:31.017585457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:37508"
ts=2026-04-20T20:38:35.876292231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
level=debug ts=2026-04-20T20:38:51.923760258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:39:05.877746121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
ts=2026-04-20T20:39:13.095921317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:44876"
ts=2026-04-20T20:39:31.022297053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:48138"
ts=2026-04-20T20:39:35.879815894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
level=debug ts=2026-04-20T20:39:38.093855447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:40:00.755539626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:51954"
ts=2026-04-20T20:40:05.882256629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
level=debug ts=2026-04-20T20:40:13.342959412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T20:40:13.344849019Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T20:40:30.757989094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:35424"
ts=2026-04-20T20:40:31.025737419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:47254"
ts=2026-04-20T20:40:35.884546627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
ts=2026-04-20T20:40:43.101556082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:57878"
ts=2026-04-20T20:41:00.760168886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:35616"
ts=2026-04-20T20:41:05.886798813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
level=debug ts=2026-04-20T20:41:13.807172844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:41:31.029856054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:39272"
ts=2026-04-20T20:41:35.888839032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
ts=2026-04-20T20:41:43.105236188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:49746"
level=error ts=2026-04-20T20:41:53.400014476Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T20:41:53.402278302Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T20:41:53.404394739Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T20:41:53.404423556Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T20:41:55.624526815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:42:05.890099157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
ts=2026-04-20T20:42:13.106788404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:35818"
ts=2026-04-20T20:42:35.891660857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
ts=2026-04-20T20:42:43.107936934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:45196"
level=debug ts=2026-04-20T20:42:46.932310899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:43:05.893740996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
ts=2026-04-20T20:43:35.895811635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
level=debug ts=2026-04-20T20:43:39.445959486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:44:01.039210267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:54416"
ts=2026-04-20T20:44:05.897939507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
level=debug ts=2026-04-20T20:44:26.40229637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T20:44:26.40394233Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T20:44:35.899187349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
ts=2026-04-20T20:45:00.775291823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:35686"
ts=2026-04-20T20:45:05.9008236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
level=debug ts=2026-04-20T20:45:09.873097122Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:45:30.77785038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:33240"
ts=2026-04-20T20:45:31.045445247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:40252"
ts=2026-04-20T20:45:35.902234055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
ts=2026-04-20T20:45:43.120524153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:32970"
ts=2026-04-20T20:46:00.780681246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:47016"
level=debug ts=2026-04-20T20:46:02.612014269Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:46:05.904264307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
ts=2026-04-20T20:46:35.906378321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
level=debug ts=2026-04-20T20:46:57.480032043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:47:05.907804857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
ts=2026-04-20T20:47:13.126257714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:55750"
ts=2026-04-20T20:47:30.786609931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:38706"
ts=2026-04-20T20:47:35.909474092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
level=debug ts=2026-04-20T20:47:40.026852105Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:48:05.91127308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
level=debug ts=2026-04-20T20:48:29.319655599Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T20:48:29.321060238Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T20:48:30.791615601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:45642"
ts=2026-04-20T20:48:31.058443982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:57074"
ts=2026-04-20T20:48:35.91344172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
ts=2026-04-20T20:48:43.131291787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:52954"
ts=2026-04-20T20:49:01.061292384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:60784"
ts=2026-04-20T20:49:05.915019295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
level=debug ts=2026-04-20T20:49:27.078738158Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:49:30.796617624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:49512"
ts=2026-04-20T20:49:35.917081675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
ts=2026-04-20T20:50:05.919356866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
ts=2026-04-20T20:50:13.137086228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:53774"
level=debug ts=2026-04-20T20:50:24.89417977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:50:35.921945509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
ts=2026-04-20T20:51:05.92398887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
ts=2026-04-20T20:51:13.14061593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:39636"
level=debug ts=2026-04-20T20:51:26.665086264Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:51:31.072019333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:35264"
ts=2026-04-20T20:51:35.926192005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
level=error ts=2026-04-20T20:51:53.402028299Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T20:51:53.404377261Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T20:51:53.406344254Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T20:51:53.406370518Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T20:52:04.678518096Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:52:05.928502181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
ts=2026-04-20T20:52:13.144421198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:40074"
ts=2026-04-20T20:52:35.930551676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
level=debug ts=2026-04-20T20:52:55.47173061Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T20:52:55.473308655Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T20:53:01.077628287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:52830"
ts=2026-04-20T20:53:05.931912033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-6jxfv-9022768a 10.129.0.74:7946"
ts=2026-04-20T20:53:13.148173561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:53984"
level=debug ts=2026-04-20T20:53:30.500703669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:53:31.079599664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56366"
ts=2026-04-20T20:53:35.933957082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
ts=2026-04-20T20:54:00.812382457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:59164"
ts=2026-04-20T20:54:05.935589298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
level=debug ts=2026-04-20T20:54:27.328773041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:54:35.937802848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
ts=2026-04-20T20:54:43.155165316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:56968"
ts=2026-04-20T20:55:01.085682912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:47058"
ts=2026-04-20T20:55:05.93990049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
level=debug ts=2026-04-20T20:55:28.029666742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:55:31.087711815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:50418"
ts=2026-04-20T20:55:35.942413437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
ts=2026-04-20T20:56:05.943979161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
level=debug ts=2026-04-20T20:56:08.736042921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T20:56:30.820999737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:53926"
ts=2026-04-20T20:56:35.945946893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-de7ed0b6 10.130.0.83:7946"
level=debug ts=2026-04-20T20:56:44.981549248Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T20:56:44.983438973Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T20:57:05.947233746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2rl78-c7dd598e 10.129.0.73:7946"
ts=2026-04-20T20:57:13.164940399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:35606"