level=info ts=2026-04-23T18:09:49.286196492Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-23T18:09:49.286234134Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-23T18:09:49.287101899Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-23T18:09:49.28740245Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-23T18:09:49.287473118Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.78 inf=eth0
level=debug ts=2026-04-23T18:09:49.288319535Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-23T18:09:49.288381319Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.78 inf=eth0
level=info ts=2026-04-23T18:09:49.288589546Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-e76bfa52
ts=2026-04-23T18:09:49.288686724Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-23T18:09:49.288709608Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.78 advertisePort=7946
level=debug ts=2026-04-23T18:09:49.288738515Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.78 advertisePort=7946
level=info ts=2026-04-23T18:09:49.288892666Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-23T18:09:49.295663283Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-23T18:09:49.295680448Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-23T18:09:49.29568951Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=6.799835ms
level=error ts=2026-04-23T18:09:49.295696348Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-23T18:09:49.295707526Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-23T18:09:49.297791669Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-23T18:09:49.297807256Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-23T18:09:49.300990851Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-23T18:09:49.301044805Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-23T18:09:49.301050082Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-23T18:09:49.301054148Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-23T18:09:49.301070748Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-04-23T18:09:49.301081959Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-23T18:09:49.301103926Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-23T18:09:49.30137812Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-23T18:09:49.301391435Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-23T18:09:49.30139522Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-23T18:09:49.301436648Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-23T18:09:49.301452703Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-23T18:09:49.30147654Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-23T18:09:49.301632694Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-23T18:09:49.301639135Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-23T18:09:49.301648401Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-23T18:09:50.302788503Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-23T18:09:50.419709841Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-23T18:09:50.419761167Z caller=loki.go:599 msg="Loki started" startup_time=1.191406766s
ts=2026-04-23T18:09:50.480773442Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-23T18:09:50.480797149Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T18:09:52.866468237Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-23T18:09:52.866494602Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-23T18:09:55.420138768Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-23T18:09:55.420200048Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-23T18:10:00.489330872Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-23T18:10:00.489356789Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-23T18:10:05.666867751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T18:10:11.0585434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:50230"
ts=2026-04-23T18:10:12.876909477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.78:7946"
ts=2026-04-23T18:10:12.877045709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:45108"
ts=2026-04-23T18:10:12.87917988Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.82:7946: dial tcp 10.130.0.82:7946: connect: connection refused"
ts=2026-04-23T18:10:12.880120061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.81:7946"
level=info ts=2026-04-23T18:10:12.881157155Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=23.58544492s
ts=2026-04-23T18:10:16.557600081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:49126"
ts=2026-04-23T18:10:16.86686059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:39202"
ts=2026-04-23T18:10:22.351765621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
ts=2026-04-23T18:10:23.59520865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:39234"
level=debug ts=2026-04-23T18:10:23.962061303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T18:10:25.864494975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:49148"
ts=2026-04-23T18:10:27.185421658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57914"
level=debug ts=2026-04-23T18:10:47.770551275Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T18:10:52.355607991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
level=debug ts=2026-04-23T18:11:05.437217396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T18:11:22.358715709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
level=debug ts=2026-04-23T18:11:32.65229092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-23T18:11:32.654008674Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T18:11:52.361783897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
level=debug ts=2026-04-23T18:12:10.506146109Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T18:12:22.364466188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:12:27.193451073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:37204"
ts=2026-04-23T18:12:52.367782417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
ts=2026-04-23T18:12:53.604898461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45880"
ts=2026-04-23T18:12:55.874631929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:38204"
ts=2026-04-23T18:13:22.370190816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:13:23.607872482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45646"
ts=2026-04-23T18:13:27.199007745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:52908"
level=debug ts=2026-04-23T18:13:30.795311694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T18:13:52.373377705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:13:57.202332236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:55506"
ts=2026-04-23T18:14:22.375980344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:14:27.206205154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:40182"
level=debug ts=2026-04-23T18:14:35.570466094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T18:14:52.37862131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
ts=2026-04-23T18:14:55.882666702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:59382"
ts=2026-04-23T18:14:57.209158953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:59202"
ts=2026-04-23T18:15:22.381671482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
ts=2026-04-23T18:15:27.212911292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:59362"
level=debug ts=2026-04-23T18:15:49.217993699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T18:15:52.385527313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
ts=2026-04-23T18:16:22.3891202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:16:25.889504546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56874"
level=debug ts=2026-04-23T18:16:44.370018231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-23T18:16:44.371664833Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T18:16:52.391981747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:17:22.395548743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:17:23.62421261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:38250"
ts=2026-04-23T18:17:27.220119691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:41054"
level=debug ts=2026-04-23T18:17:36.302902497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:17:52.398948833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
level=debug ts=2026-04-23T18:18:22.167551861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:18:22.402680994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:18:27.225539975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57618"
ts=2026-04-23T18:18:52.406458179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:18:55.900265645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:55210"
ts=2026-04-23T18:18:57.229301882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:38490"
level=debug ts=2026-04-23T18:19:03.432954021Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:19:22.409328712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
level=debug ts=2026-04-23T18:19:39.868922201Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:19:52.412231615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:19:53.634195694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:43692"
level=info ts=2026-04-23T18:19:55.42043554Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-23T18:19:55.424403981Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T18:19:55.42623537Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T18:19:55.428087093Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T18:19:55.42813094Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-23T18:19:55.428144553Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-23T18:20:22.415319271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
level=debug ts=2026-04-23T18:20:29.950297486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T18:20:29.951906411Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T18:20:52.419108869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
level=debug ts=2026-04-23T18:21:12.832395512Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:21:22.422382587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:21:52.426206641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:21:53.643042326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:54858"
ts=2026-04-23T18:21:57.242612199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:39066"
level=debug ts=2026-04-23T18:22:07.349249694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:22:22.429480893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
ts=2026-04-23T18:22:23.646683993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:36026"
level=debug ts=2026-04-23T18:22:47.398612395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:22:52.433493715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:23:22.436710582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
level=debug ts=2026-04-23T18:23:40.073228298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:23:52.439412875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:23:57.251836424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:40024"
ts=2026-04-23T18:24:22.442576881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:24:23.656623431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:56860"
ts=2026-04-23T18:24:27.255792588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:33874"
level=debug ts=2026-04-23T18:24:40.417846614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T18:24:40.419529537Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T18:24:52.445230598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
level=debug ts=2026-04-23T18:25:20.073599183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:25:22.448657423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:25:23.661388507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:40828"
ts=2026-04-23T18:25:52.452477619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:25:55.929088528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56490"
ts=2026-04-23T18:25:57.261941173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:43260"
level=debug ts=2026-04-23T18:26:05.165193324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:26:22.455757159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
ts=2026-04-23T18:26:25.931785955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:51204"
ts=2026-04-23T18:26:27.265183312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:44356"
level=debug ts=2026-04-23T18:26:40.824303683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:26:52.458416529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:26:55.934623058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:39660"
ts=2026-04-23T18:26:57.268039921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:58278"
ts=2026-04-23T18:27:22.461592893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
level=debug ts=2026-04-23T18:27:28.301882789Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:27:52.464618899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
level=debug ts=2026-04-23T18:28:04.399752408Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T18:28:04.401707102Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T18:28:22.468097707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:28:23.671130482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:56310"
ts=2026-04-23T18:28:25.942125437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:35766"
ts=2026-04-23T18:28:52.471065815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:28:55.945361084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:35404"
level=debug ts=2026-04-23T18:29:03.988613382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:29:22.474329423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
ts=2026-04-23T18:29:25.94830863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:35970"
level=debug ts=2026-04-23T18:29:47.939235762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:29:52.477363783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
level=error ts=2026-04-23T18:29:55.431423446Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T18:29:55.433644012Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T18:29:55.435454355Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T18:29:55.435481798Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T18:29:55.950868313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:60002"
ts=2026-04-23T18:29:57.282196948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:38916"
ts=2026-04-23T18:30:22.480970538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
ts=2026-04-23T18:30:23.682066202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:37840"
level=debug ts=2026-04-23T18:30:27.003222918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:30:27.2858506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:46068"
ts=2026-04-23T18:30:52.485158891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
level=debug ts=2026-04-23T18:31:11.263257319Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:31:22.487938483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:31:27.290730074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:53880"
level=debug ts=2026-04-23T18:31:49.063781955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T18:31:49.066618669Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T18:31:52.490723512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:31:53.689514439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:47478"
ts=2026-04-23T18:31:55.959188497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56480"
ts=2026-04-23T18:32:22.493892378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:32:27.2964905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:52320"
level=debug ts=2026-04-23T18:32:37.242893229Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:32:52.497594861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:32:55.964269003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:38362"
ts=2026-04-23T18:33:22.500948788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:33:25.967515134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56036"
ts=2026-04-23T18:33:27.301528327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57648"
level=debug ts=2026-04-23T18:33:29.842449323Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:33:52.504645139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:33:55.970301913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:58154"
level=debug ts=2026-04-23T18:34:08.660846024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:34:22.508072012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
ts=2026-04-23T18:34:25.973472086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:49118"
ts=2026-04-23T18:34:52.515511635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:34:57.309148068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:47998"
level=debug ts=2026-04-23T18:35:10.283686499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:35:22.518679099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:35:25.982481293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:60406"
ts=2026-04-23T18:35:52.521379835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
level=debug ts=2026-04-23T18:36:11.191827656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T18:36:11.193579136Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T18:36:22.524733395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:36:23.70798985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:48572"
ts=2026-04-23T18:36:52.527211438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:36:53.711885143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:53168"
ts=2026-04-23T18:36:57.321772683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:45492"
level=debug ts=2026-04-23T18:37:09.956227123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:37:22.5332191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
level=debug ts=2026-04-23T18:37:45.862577227Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:37:52.537801625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:37:55.996510858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:49770"
ts=2026-04-23T18:38:22.541339096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
ts=2026-04-23T18:38:25.99911327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:43136"
level=debug ts=2026-04-23T18:38:46.67914889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:38:52.544825694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:38:53.720846916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:56626"
ts=2026-04-23T18:38:56.001944874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:42658"
ts=2026-04-23T18:39:22.547737125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:39:27.335223171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:55754"
level=debug ts=2026-04-23T18:39:45.483459618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:39:52.550607177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
level=error ts=2026-04-23T18:39:55.431378313Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T18:39:55.43394505Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T18:39:55.436322046Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T18:39:55.43636559Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T18:39:57.338800427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:55468"
level=debug ts=2026-04-23T18:40:19.671747205Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T18:40:19.673447664Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T18:40:22.557003831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
ts=2026-04-23T18:40:23.727145328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:59602"
ts=2026-04-23T18:40:27.342707498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:45808"
ts=2026-04-23T18:40:52.559844605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:40:56.011531079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:43222"
ts=2026-04-23T18:40:57.347340659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:39348"
level=debug ts=2026-04-23T18:41:03.61039487Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:41:22.563095489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:41:27.351223891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:56040"
level=debug ts=2026-04-23T18:41:51.666011955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:41:52.56705907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:41:56.016289983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:34460"
ts=2026-04-23T18:42:22.570548131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b6f45646 10.128.0.76:7946"
ts=2026-04-23T18:42:23.734830036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:43520"
ts=2026-04-23T18:42:27.355471327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:40660"
level=debug ts=2026-04-23T18:42:37.463650231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:42:52.573615081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:42:56.020533448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:43804"
ts=2026-04-23T18:42:57.385175673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:46042"
ts=2026-04-23T18:42:57.395573787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:46050"
level=debug ts=2026-04-23T18:43:16.183611667Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:43:16.289967089Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b6f45646 (timeout reached)"
ts=2026-04-23T18:43:19.290174908Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b6f45646 has failed, no acks received"
ts=2026-04-23T18:43:22.577372493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:43:24.894827028Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-b6f45646 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T18:43:26.023919501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:34268"
ts=2026-04-23T18:43:52.58121345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:43:56.027589289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56532"
level=debug ts=2026-04-23T18:44:13.113936763Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T18:44:13.115461523Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T18:44:22.585028602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:44:52.587873349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:44:53.745892554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:60634"
level=debug ts=2026-04-23T18:45:07.535597396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:45:12.740003601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:46580"
ts=2026-04-23T18:45:22.591077801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b300f8ff 10.128.0.76:7946"
ts=2026-04-23T18:45:23.749903023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:36174"
level=debug ts=2026-04-23T18:45:50.204765929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:45:52.59422849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b300f8ff 10.128.0.76:7946"
ts=2026-04-23T18:45:56.039706091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:46092"
ts=2026-04-23T18:46:12.746724259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:53254"
ts=2026-04-23T18:46:22.597389844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b300f8ff 10.128.0.76:7946"
ts=2026-04-23T18:46:26.045713035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:42940"
level=debug ts=2026-04-23T18:46:26.765175419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:46:52.600436438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b300f8ff 10.128.0.76:7946"
level=debug ts=2026-04-23T18:47:02.063393545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:47:22.603741396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:47:26.057739733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:54728"
ts=2026-04-23T18:47:52.606898775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b300f8ff 10.128.0.76:7946"
level=debug ts=2026-04-23T18:47:54.988709947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T18:47:54.990228167Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T18:48:22.610081203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:48:23.762033627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:54424"
level=debug ts=2026-04-23T18:48:51.37324656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:48:52.612963261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:48:53.765598079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:39418"
ts=2026-04-23T18:49:12.759017053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:46602"
ts=2026-04-23T18:49:22.616270311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b300f8ff 10.128.0.76:7946"
level=debug ts=2026-04-23T18:49:45.403635837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:49:52.619054091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
level=error ts=2026-04-23T18:49:55.432880974Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T18:49:55.435349596Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T18:49:55.437481092Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T18:49:55.437514947Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T18:50:12.764904932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:47406"
ts=2026-04-23T18:50:22.622624174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
level=debug ts=2026-04-23T18:50:43.242462551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:50:52.625970212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:51:06.290581246Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b300f8ff (timeout reached)"
ts=2026-04-23T18:51:09.290747042Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b300f8ff has failed, no acks received"
ts=2026-04-23T18:51:16.29047733Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b300f8ff (timeout reached)"
ts=2026-04-23T18:51:18.425701674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:33378"
ts=2026-04-23T18:51:18.439042401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:33384"
ts=2026-04-23T18:51:19.289938626Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b300f8ff has failed, no acks received"
ts=2026-04-23T18:51:22.629040089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0e2ffe16 10.128.0.76:7946"
ts=2026-04-23T18:51:29.291023827Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-b300f8ff as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-23T18:51:43.790161872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:51:52.63872095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0e2ffe16 10.128.0.76:7946"
ts=2026-04-23T18:51:56.079220199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48736"
ts=2026-04-23T18:52:17.486844929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:43312"
ts=2026-04-23T18:52:22.641743678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
level=debug ts=2026-04-23T18:52:27.705818023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T18:52:27.707689279Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T18:52:47.490269211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:38596"
ts=2026-04-23T18:52:52.644592829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:52:56.084219918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48334"
level=debug ts=2026-04-23T18:53:05.606551266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:53:22.647415195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0e2ffe16 10.128.0.76:7946"
ts=2026-04-23T18:53:52.651220527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0e2ffe16 10.128.0.76:7946"
level=debug ts=2026-04-23T18:54:01.003454121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:54:22.654008403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0e2ffe16 10.128.0.76:7946"
ts=2026-04-23T18:54:47.498852229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:43926"
level=debug ts=2026-04-23T18:54:51.190302111Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:54:52.657135553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0e2ffe16 10.128.0.76:7946"
ts=2026-04-23T18:54:53.794912238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:47126"
ts=2026-04-23T18:55:22.660689312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
level=debug ts=2026-04-23T18:55:39.114092832Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:55:52.664121592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0e2ffe16 10.128.0.76:7946"
ts=2026-04-23T18:56:17.50591722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:45026"
ts=2026-04-23T18:56:22.667227053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
level=debug ts=2026-04-23T18:56:36.978712707Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T18:56:36.980453811Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T18:56:52.670241087Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
level=debug ts=2026-04-23T18:57:14.000423556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:57:22.672716565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:57:26.10157228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:60724"
ts=2026-04-23T18:57:52.675456842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:57:53.805311267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:51472"
level=debug ts=2026-04-23T18:58:12.17953757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:58:17.515454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57054"
ts=2026-04-23T18:58:22.678393575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:58:23.80890945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:51276"
ts=2026-04-23T18:58:26.111427458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:34378"
ts=2026-04-23T18:58:46.290231239Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-0e2ffe16 (timeout reached)"
ts=2026-04-23T18:58:49.290524641Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-0e2ffe16 has failed, no acks received"
ts=2026-04-23T18:58:52.681269542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T18:58:53.811719112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:42272"
level=debug ts=2026-04-23T18:58:54.973421949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T18:59:01.289971417Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-0e2ffe16 (timeout reached)"
ts=2026-04-23T18:59:04.289545597Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-0e2ffe16 has failed, no acks received"
ts=2026-04-23T18:59:09.290830634Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-0e2ffe16 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T18:59:14.394402279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:39228"
ts=2026-04-23T18:59:14.407211824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:39230"
ts=2026-04-23T18:59:22.684326848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T18:59:52.68775211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ee2c2629 10.128.0.76:7946"
ts=2026-04-23T18:59:53.817131636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:50360"
level=debug ts=2026-04-23T18:59:54.576987604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T18:59:55.430732724Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T18:59:55.432689019Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T18:59:55.434703568Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T18:59:55.434737117Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T19:00:22.691439959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:00:23.820801404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:50528"
level=debug ts=2026-04-23T19:00:40.711052336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T19:00:40.712858348Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T19:00:52.694617098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:00:56.12517909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:41192"
level=debug ts=2026-04-23T19:01:20.636132706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:01:22.697981922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:01:52.700776796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
level=debug ts=2026-04-23T19:02:00.655604949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:02:22.70400507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:02:23.828664994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:41292"
ts=2026-04-23T19:02:52.70729887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:02:53.832244835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:37278"
level=debug ts=2026-04-23T19:02:54.893381582Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:03:22.709758475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:03:26.136234902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:46700"
level=debug ts=2026-04-23T19:03:31.549198664Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:03:36.460405929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:51764"
ts=2026-04-23T19:03:52.713183941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:03:53.837019267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:38892"
level=debug ts=2026-04-23T19:04:10.638419279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T19:04:10.639953616Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T19:04:22.716241582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
level=debug ts=2026-04-23T19:04:49.99562622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:04:52.718823229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:04:56.141914054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:34206"
ts=2026-04-23T19:05:22.722333061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ee2c2629 10.128.0.76:7946"
level=debug ts=2026-04-23T19:05:26.195788193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:05:52.725866357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:05:56.14686377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:47250"
ts=2026-04-23T19:06:06.471324867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:46860"
level=debug ts=2026-04-23T19:06:12.552498258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:06:22.728786903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:06:23.847430202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45506"
ts=2026-04-23T19:06:52.732487562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:06:53.850931527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:52546"
ts=2026-04-23T19:06:56.151596095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:53130"
level=debug ts=2026-04-23T19:07:05.652717293Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:07:06.477083713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:34942"
ts=2026-04-23T19:07:22.73566223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:07:26.154888289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:47304"
ts=2026-04-23T19:07:36.480842542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:38764"
level=debug ts=2026-04-23T19:07:49.300124385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T19:07:49.301999797Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T19:07:52.738851712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ee2c2629 10.128.0.76:7946"
ts=2026-04-23T19:07:53.856872697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:35720"
ts=2026-04-23T19:07:56.157899991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56140"
ts=2026-04-23T19:08:22.742692476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:08:46.289602437Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-ee2c2629 (timeout reached)"
level=debug ts=2026-04-23T19:08:47.063437949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:08:49.289841753Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-ee2c2629 has failed, no acks received"
ts=2026-04-23T19:08:52.746744973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:08:53.861598927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:57620"
ts=2026-04-23T19:08:56.163404337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:42280"
ts=2026-04-23T19:08:59.894133064Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-ee2c2629 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T19:09:01.290599998Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-ee2c2629 (timeout reached)"
ts=2026-04-23T19:09:04.290183282Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-ee2c2629 has failed, no acks received"
ts=2026-04-23T19:09:17.392247697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57672"
ts=2026-04-23T19:09:17.403223354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57680"
ts=2026-04-23T19:09:22.749906563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
level=debug ts=2026-04-23T19:09:27.892542965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:09:52.752686026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:09:53.867102049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:34286"
level=error ts=2026-04-23T19:09:55.435548128Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T19:09:55.4377404Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T19:09:55.439714726Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T19:09:55.439740477Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-23T19:10:15.736366264Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:10:22.7562253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-df5d1c29 10.128.0.76:7946"
ts=2026-04-23T19:10:50.324136529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:34982"
ts=2026-04-23T19:10:52.759669322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:10:53.871742678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:54466"
level=debug ts=2026-04-23T19:10:54.658271313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:10:56.172643983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:54302"
ts=2026-04-23T19:11:20.327246451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:45538"
ts=2026-04-23T19:11:22.763059862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:11:23.875299241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:44724"
level=debug ts=2026-04-23T19:11:32.658667469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T19:11:32.660369178Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T19:11:52.766411582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-df5d1c29 10.128.0.76:7946"
ts=2026-04-23T19:11:56.177459985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:45460"
ts=2026-04-23T19:12:20.333395443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:60600"
ts=2026-04-23T19:12:22.770232348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:12:23.879249306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:38180"
level=debug ts=2026-04-23T19:12:32.030161161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:12:52.773279376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-df5d1c29 10.128.0.76:7946"
ts=2026-04-23T19:12:53.882375969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:33586"
ts=2026-04-23T19:12:56.182397108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:47820"
level=debug ts=2026-04-23T19:13:19.738195438Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:13:22.776985052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-df5d1c29 10.128.0.76:7946"
ts=2026-04-23T19:13:26.185685124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56502"
ts=2026-04-23T19:13:52.780088056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-df5d1c29 10.128.0.76:7946"
ts=2026-04-23T19:13:53.887262515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:41678"
ts=2026-04-23T19:13:56.188505363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:45810"
level=debug ts=2026-04-23T19:14:10.669295582Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:14:20.342297642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:42810"
ts=2026-04-23T19:14:22.783321706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:14:26.191536773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:43310"
level=debug ts=2026-04-23T19:14:45.622982656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:14:52.786112812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-df5d1c29 10.128.0.76:7946"
ts=2026-04-23T19:14:53.892986424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:57316"
ts=2026-04-23T19:15:22.789778402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-df5d1c29 10.128.0.76:7946"
ts=2026-04-23T19:15:23.896457656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45638"
ts=2026-04-23T19:15:26.196149562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:40334"
level=debug ts=2026-04-23T19:15:27.086194066Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T19:15:27.08818345Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T19:15:52.793025324Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-df5d1c29 failed: dial tcp 10.128.0.76:7946: connect: connection refused"
ts=2026-04-23T19:15:53.90030245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:40004"
ts=2026-04-23T19:15:56.198813956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:43302"
ts=2026-04-23T19:16:01.289481402Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-df5d1c29 (timeout reached)"
ts=2026-04-23T19:16:04.289694892Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-df5d1c29 has failed, no acks received"
level=debug ts=2026-04-23T19:16:05.258405275Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:16:10.382379255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57324"
ts=2026-04-23T19:16:10.394177278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57336"
ts=2026-04-23T19:16:16.289296893Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-df5d1c29 (timeout reached)"
ts=2026-04-23T19:16:19.289758715Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-df5d1c29 has failed, no acks received"
ts=2026-04-23T19:16:22.795129284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:16:23.903911445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:50510"
ts=2026-04-23T19:16:26.201722075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:53456"
ts=2026-04-23T19:16:45.945365279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:54486"
ts=2026-04-23T19:16:52.798573666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:16:53.907373954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:56656"
level=debug ts=2026-04-23T19:16:58.005291069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:17:22.801285901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:17:23.910639402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:40730"
level=debug ts=2026-04-23T19:17:41.73051913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:17:52.804217828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b290d013 10.128.0.76:7946"
ts=2026-04-23T19:17:53.913542764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:41294"
ts=2026-04-23T19:17:56.209442132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:57008"
level=debug ts=2026-04-23T19:18:22.525038543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:18:22.807054834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:18:26.212679238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:41100"
ts=2026-04-23T19:18:52.810140508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:18:56.215296037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:46712"
level=debug ts=2026-04-23T19:19:11.431984052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T19:19:11.433795135Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T19:19:22.8134224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b290d013 10.128.0.76:7946"
ts=2026-04-23T19:19:45.958698912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:35014"
ts=2026-04-23T19:19:52.817149154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:19:53.922436444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:53082"
level=error ts=2026-04-23T19:19:55.434235497Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T19:19:55.436117632Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T19:19:55.437751828Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T19:19:55.437783759Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-23T19:20:00.036907468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:20:22.820833043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:20:26.222934756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:42292"
level=debug ts=2026-04-23T19:20:50.410464011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:20:52.824352792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:20:56.22603754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:43228"
ts=2026-04-23T19:21:22.827593232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b290d013 10.128.0.76:7946"
level=debug ts=2026-04-23T19:21:43.494957192Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:21:52.830811262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:22:15.968293694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:52484"
ts=2026-04-23T19:22:22.833365592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:22:26.232361688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:43904"
level=debug ts=2026-04-23T19:22:30.515093802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:22:52.836373665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b290d013 10.128.0.76:7946"
ts=2026-04-23T19:22:53.933818898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:54576"
level=debug ts=2026-04-23T19:23:11.691615717Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T19:23:11.693198037Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T19:23:22.840122632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:23:23.937522408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:33992"
ts=2026-04-23T19:23:52.842900783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b290d013 10.128.0.76:7946"
ts=2026-04-23T19:23:56.239866271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56056"
level=debug ts=2026-04-23T19:24:03.490032657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:24:22.846224004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:24:23.94276185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:37440"
level=debug ts=2026-04-23T19:24:43.019045762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:24:51.290241502Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b290d013 (timeout reached)"
ts=2026-04-23T19:24:52.849433142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:24:54.289919591Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b290d013 has failed, no acks received"
ts=2026-04-23T19:25:11.289343479Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b290d013 (timeout reached)"
ts=2026-04-23T19:25:14.290017886Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b290d013 has failed, no acks received"
ts=2026-04-23T19:25:14.290151689Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-b290d013 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T19:25:22.852246914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:25:23.377729312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:49576"
ts=2026-04-23T19:25:23.388914715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:49580"
level=debug ts=2026-04-23T19:25:33.006549048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:25:52.855180441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cd0f7b01 10.128.0.76:7946"
ts=2026-04-23T19:26:05.916835462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:59692"
level=debug ts=2026-04-23T19:26:10.766226633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:26:22.858953273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cd0f7b01 10.128.0.76:7946"
ts=2026-04-23T19:26:35.920626666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:38608"
level=debug ts=2026-04-23T19:26:52.465126789Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T19:26:52.466731556Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T19:26:52.862202678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:27:05.924336985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:34070"
ts=2026-04-23T19:27:22.86510134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
level=debug ts=2026-04-23T19:27:42.042644489Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:27:52.868367332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:27:53.956493835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:60632"
ts=2026-04-23T19:27:56.258403372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:45248"
ts=2026-04-23T19:28:05.929398932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:39932"
level=debug ts=2026-04-23T19:28:20.937328037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:28:22.871899179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cd0f7b01 10.128.0.76:7946"
ts=2026-04-23T19:28:23.959317361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:58496"
ts=2026-04-23T19:28:52.875489124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:28:53.962672416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:47672"
level=debug ts=2026-04-23T19:28:58.353123602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:29:22.879021519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cd0f7b01 10.128.0.76:7946"
ts=2026-04-23T19:29:23.966460788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:40474"
ts=2026-04-23T19:29:26.265778845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56188"
ts=2026-04-23T19:29:35.93584256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:60084"
level=debug ts=2026-04-23T19:29:43.068141756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:29:52.881841929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cd0f7b01 10.128.0.76:7946"
level=error ts=2026-04-23T19:29:55.432584282Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T19:29:55.434768894Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T19:29:55.436635288Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T19:29:55.436665707Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T19:30:22.885540807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cd0f7b01 10.128.0.76:7946"
ts=2026-04-23T19:30:26.270118494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:36756"
ts=2026-04-23T19:30:35.940524172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:52680"
level=debug ts=2026-04-23T19:30:37.785717425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T19:30:37.787247994Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T19:30:52.889388375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:31:22.891831125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
level=debug ts=2026-04-23T19:31:30.908142938Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:31:35.945955297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:53520"
ts=2026-04-23T19:31:52.894915972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cd0f7b01 10.128.0.76:7946"
ts=2026-04-23T19:31:56.277336646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:43126"
ts=2026-04-23T19:32:05.949474434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:44328"
ts=2026-04-23T19:32:22.898463275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:32:26.280418893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:54162"
level=debug ts=2026-04-23T19:32:26.904018538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:32:46.290123722Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-cd0f7b01 (timeout reached)"
ts=2026-04-23T19:32:49.289734024Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-cd0f7b01 has failed, no acks received"
ts=2026-04-23T19:32:52.901265031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:32:56.283644935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:50474"
ts=2026-04-23T19:32:56.290155416Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-cd0f7b01 (timeout reached)"
ts=2026-04-23T19:32:58.454058283Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-cd0f7b01 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T19:32:59.28920114Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-cd0f7b01 has failed, no acks received"
ts=2026-04-23T19:33:17.38269589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:36332"
ts=2026-04-23T19:33:17.393318574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:36348"
ts=2026-04-23T19:33:22.904536649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
level=debug ts=2026-04-23T19:33:25.92440898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:33:26.286465862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:38074"
ts=2026-04-23T19:33:52.907447257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-60d8a0f5 10.128.0.76:7946"
level=debug ts=2026-04-23T19:34:03.803766298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:34:16.318795856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:33424"
ts=2026-04-23T19:34:22.910638032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:34:26.291588946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:40206"
ts=2026-04-23T19:34:46.322119856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:47150"
level=debug ts=2026-04-23T19:34:47.579137476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T19:34:47.580876707Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T19:34:52.913728573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:34:53.984362702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:42002"
ts=2026-04-23T19:34:56.294863572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:37830"
ts=2026-04-23T19:35:22.918028416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-60d8a0f5 10.128.0.76:7946"
ts=2026-04-23T19:35:23.987765544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:38426"
level=debug ts=2026-04-23T19:35:26.775144148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:35:52.92129349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-60d8a0f5 10.128.0.76:7946"
ts=2026-04-23T19:35:53.991467704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:57200"
level=debug ts=2026-04-23T19:36:06.750769062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:36:22.924410286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
level=debug ts=2026-04-23T19:36:52.75792778Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:36:52.927915669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-60d8a0f5 10.128.0.76:7946"
ts=2026-04-23T19:37:22.931904702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:37:23.999023159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:52806"
level=debug ts=2026-04-23T19:37:31.585693962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:37:52.93517777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:38:22.938110395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-60d8a0f5 10.128.0.76:7946"
ts=2026-04-23T19:38:24.003997975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:50558"
level=debug ts=2026-04-23T19:38:24.174260931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T19:38:24.176019031Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T19:38:26.310285354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:37424"
ts=2026-04-23T19:38:52.941096563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
level=debug ts=2026-04-23T19:39:15.524058426Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:39:22.944213413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:39:26.315397185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:59766"
ts=2026-04-23T19:39:52.94775342Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-60d8a0f5 failed: dial tcp 10.128.0.76:7946: connect: connection refused"
level=error ts=2026-04-23T19:39:55.435735418Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T19:39:55.437679162Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T19:39:55.439528004Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T19:39:55.439553156Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T19:39:56.318222098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:37192"
level=debug ts=2026-04-23T19:39:58.667290225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:40:01.290158642Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-60d8a0f5 (timeout reached)"
ts=2026-04-23T19:40:04.290133447Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-60d8a0f5 has failed, no acks received"
ts=2026-04-23T19:40:16.290594516Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-60d8a0f5 (timeout reached)"
ts=2026-04-23T19:40:19.290388259Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-60d8a0f5 has failed, no acks received"
ts=2026-04-23T19:40:22.950191443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:40:23.45514771Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-60d8a0f5 as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-23T19:40:39.691313508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:40:52.95405447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:41:13.372546434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:55260"
ts=2026-04-23T19:41:13.382494933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:55272"
level=debug ts=2026-04-23T19:41:14.340861526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:41:22.957840755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-265fc6ee 10.128.0.76:7946"
ts=2026-04-23T19:41:24.015740262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:41756"
ts=2026-04-23T19:41:26.324922858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:58446"
ts=2026-04-23T19:41:52.961476444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-265fc6ee 10.128.0.76:7946"
ts=2026-04-23T19:41:55.376154646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:33476"
level=debug ts=2026-04-23T19:42:01.775099462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T19:42:01.776690969Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T19:42:22.964201437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:42:25.37972347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:47910"
ts=2026-04-23T19:42:41.290400779Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-265fc6ee (timeout reached)"
ts=2026-04-23T19:42:44.289479549Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-265fc6ee has failed, no acks received"
level=debug ts=2026-04-23T19:42:50.094721837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:42:52.96710428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:42:54.021442779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45504"
ts=2026-04-23T19:43:01.290241457Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-265fc6ee (timeout reached)"
ts=2026-04-23T19:43:04.290153675Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-265fc6ee as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T19:43:04.290187245Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-265fc6ee has failed, no acks received"
ts=2026-04-23T19:43:22.970307954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:43:24.024912331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:39462"
level=debug ts=2026-04-23T19:43:31.013741325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:43:52.972925743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:43:54.028041023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:39736"
ts=2026-04-23T19:43:56.336761888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:52084"
level=debug ts=2026-04-23T19:44:07.359026093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:44:22.975865918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-t82q5-84546b86 10.128.0.73:7946"
ts=2026-04-23T19:44:26.339312374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:50094"
level=debug ts=2026-04-23T19:44:52.080292292Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T19:44:52.979166072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"
ts=2026-04-23T19:44:54.032846532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:60866"
ts=2026-04-23T19:45:19.373352678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:56398"
ts=2026-04-23T19:45:19.385409135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:56410"
ts=2026-04-23T19:45:22.981954864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-m89vl-985f406b 10.130.0.81:7946"