level=info ts=2026-04-22T18:59:38.233550876Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-22T18:59:38.233623165Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-22T18:59:38.236283592Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-22T18:59:38.236691794Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T18:59:38.236799888Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.70 inf=eth0
level=debug ts=2026-04-22T18:59:38.237335663Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T18:59:38.237438642Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.70 inf=eth0
level=info ts=2026-04-22T18:59:38.237938077Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-631d9765
ts=2026-04-22T18:59:38.238027577Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-22T18:59:38.238043411Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.70 advertisePort=7946
level=debug ts=2026-04-22T18:59:38.238070483Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.70 advertisePort=7946
level=info ts=2026-04-22T18:59:38.23822591Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-22T18:59:38.245945165Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T18:59:38.24596208Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T18:59:38.245971485Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=7.748217ms
level=error ts=2026-04-22T18:59:38.245978434Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-22T18:59:38.245989944Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-22T18:59:38.250639175Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T18:59:38.250657862Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T18:59:38.254508726Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-04-22T18:59:38.254655577Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-22T18:59:38.254668986Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-22T18:59:38.255108768Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-22T18:59:38.255340854Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-22T18:59:38.255445618Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-22T18:59:38.255491424Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-22T18:59:38.25550685Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-22T18:59:38.255515589Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-22T18:59:38.255521024Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-22T18:59:38.255572795Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-22T18:59:38.255628335Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-22T18:59:38.255647817Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-22T18:59:38.255781316Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-22T18:59:38.431380876Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-22T18:59:38.431487573Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-22T18:59:39.432476947Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
ts=2026-04-22T18:59:39.480235582Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T18:59:39.480270018Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T18:59:39.546396113Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-22T18:59:39.546467733Z caller=loki.go:599 msg="Loki started" startup_time=1.384579994s
ts=2026-04-22T18:59:41.483851556Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T18:59:41.483875201Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T18:59:44.547441405Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-22T18:59:44.547510911Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-22T18:59:48.374507486Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T18:59:48.374530853Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T18:59:54.892373496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:00:00.641736408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:51602"
ts=2026-04-22T19:00:01.302702687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:37130"
ts=2026-04-22T19:00:01.817113512Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.63:7946: dial tcp 10.130.0.63:7946: connect: connection refused"
ts=2026-04-22T19:00:01.81786374Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.62:7946: dial tcp 10.130.0.62:7946: connect: connection refused"
ts=2026-04-22T19:00:01.818026767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.61:7946"
ts=2026-04-22T19:00:01.818657484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.65:7946"
ts=2026-04-22T19:00:01.819997045Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.68:7946: dial tcp 10.130.0.68:7946: connect: connection refused"
ts=2026-04-22T19:00:01.820081839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.70:7946"
ts=2026-04-22T19:00:01.82010868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:38936"
level=info ts=2026-04-22T19:00:01.820539416Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=23.574543434s
ts=2026-04-22T19:00:05.843473527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:41788"
level=debug ts=2026-04-22T19:00:12.242169987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:00:15.22224147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c92dbcf 10.130.0.69:7946"
ts=2026-04-22T19:00:23.4841595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:40662"
ts=2026-04-22T19:00:33.940091691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:36826"
level=debug ts=2026-04-22T19:00:34.530293887Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:00:45.224052456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T19:00:59.305948779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:01:15.22533356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T19:01:23.681317175Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T19:01:23.682912549Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:01:44.418244008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33004"
ts=2026-04-22T19:01:45.226956917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T19:02:06.980090397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:02:15.228628868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:02:45.230055938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T19:02:58.883772043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:03:03.946886428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:59896"
ts=2026-04-22T19:03:15.231566849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c92dbcf 10.130.0.69:7946"
ts=2026-04-22T19:03:45.232634541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T19:03:50.861607422Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:04:14.424645311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41098"
ts=2026-04-22T19:04:15.233605157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:04:33.951025859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:36668"
ts=2026-04-22T19:04:44.426312303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42692"
ts=2026-04-22T19:04:45.23528796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c92dbcf 10.130.0.69:7946"
level=debug ts=2026-04-22T19:05:05.059961786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:05:15.236492602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:05:23.496446242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:54144"
ts=2026-04-22T19:05:45.237502866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c92dbcf 10.130.0.69:7946"
ts=2026-04-22T19:06:15.238568997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c92dbcf 10.130.0.69:7946"
level=debug ts=2026-04-22T19:06:22.161142862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T19:06:22.162724224Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:06:23.498921984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:51204"
ts=2026-04-22T19:06:45.240068394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:07:15.241174836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c92dbcf 10.130.0.69:7946"
level=debug ts=2026-04-22T19:07:23.27001603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:07:45.242252886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:08:03.96134823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57394"
level=debug ts=2026-04-22T19:08:04.82161318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:08:15.243692782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c92dbcf 10.130.0.69:7946"
ts=2026-04-22T19:08:33.962489134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55714"
level=debug ts=2026-04-22T19:08:42.018944074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:08:44.436678874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39764"
ts=2026-04-22T19:08:45.244622506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:09:14.437695754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45690"
ts=2026-04-22T19:09:15.245474999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:09:23.506649479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:37556"
ts=2026-04-22T19:09:33.964663789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:44194"
level=debug ts=2026-04-22T19:09:34.628130329Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T19:09:44.547972175Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-22T19:09:44.550550504Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:09:44.552466242Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:09:44.554207834Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:09:44.554245498Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T19:09:44.554259509Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-22T19:09:45.246493469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c92dbcf 10.130.0.69:7946"
ts=2026-04-22T19:10:14.440811535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46986"
ts=2026-04-22T19:10:15.247979012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:10:23.509164909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:44466"
level=debug ts=2026-04-22T19:10:23.697270212Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:10:23.69893437Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:10:33.967777047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:38824"
ts=2026-04-22T19:10:44.44232467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:35024"
ts=2026-04-22T19:10:45.249232735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c92dbcf 10.130.0.69:7946"
ts=2026-04-22T19:11:03.969240378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:54846"
ts=2026-04-22T19:11:14.443921455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42460"
ts=2026-04-22T19:11:15.250770548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T19:11:18.376897047Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:11:44.444878016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51112"
ts=2026-04-22T19:11:45.251741388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T19:12:09.087660543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:12:14.446053015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:38250"
ts=2026-04-22T19:12:15.252955912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:12:23.513400381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:35908"
ts=2026-04-22T19:12:45.254004897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T19:12:49.906479836Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:12:53.514843988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:34296"
ts=2026-04-22T19:13:03.974925664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:46472"
ts=2026-04-22T19:13:15.255816133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:13:23.51611303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:47490"
level=debug ts=2026-04-22T19:13:24.929693674Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:13:44.449637083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40408"
ts=2026-04-22T19:13:45.25744703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:14:15.258726298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c92dbcf 10.130.0.69:7946"
level=debug ts=2026-04-22T19:14:18.204875969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:14:18.206467464Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:14:45.260544039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:14:53.519485373Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55422"
level=debug ts=2026-04-22T19:15:13.804925081Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:15:15.261847989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:15:23.521246122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:38742"
ts=2026-04-22T19:15:45.263231284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c92dbcf 10.130.0.69:7946"
level=debug ts=2026-04-22T19:15:50.263022617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:16:03.983300002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:59178"
ts=2026-04-22T19:16:15.264289576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T19:16:33.632465368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:16:33.985017909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:60954"
ts=2026-04-22T19:16:45.265346343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:16:53.524881216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:54456"
ts=2026-04-22T19:17:03.986283729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:59320"
ts=2026-04-22T19:17:15.266959667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c92dbcf 10.130.0.69:7946"
level=debug ts=2026-04-22T19:17:19.581729228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:17:23.525891332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:57550"
ts=2026-04-22T19:17:33.987465584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:38894"
ts=2026-04-22T19:17:44.460646042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36618"
ts=2026-04-22T19:17:45.26839719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T19:18:08.444451136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:18:08.446472067Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:18:14.462034805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46796"
ts=2026-04-22T19:18:15.26915359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c92dbcf 10.130.0.69:7946"
ts=2026-04-22T19:18:33.990516956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:42766"
ts=2026-04-22T19:18:45.270436438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T19:18:52.886812988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:19:14.465246629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40330"
ts=2026-04-22T19:19:15.271519718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c92dbcf 10.130.0.69:7946"
level=debug ts=2026-04-22T19:19:27.537864546Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:19:44.466872077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43698"
level=error ts=2026-04-22T19:19:44.558088141Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:19:44.559908026Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:19:44.561779975Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:19:44.561804281Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:19:45.272818096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:20:14.467961188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36940"
ts=2026-04-22T19:20:15.2739209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:20:23.538260393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:33532"
level=debug ts=2026-04-22T19:20:25.838819691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:20:44.473989887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33288"
ts=2026-04-22T19:20:45.275009932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c92dbcf 10.130.0.69:7946"
ts=2026-04-22T19:20:53.539459313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:42812"
ts=2026-04-22T19:21:15.276731264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T19:21:24.157762511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:21:33.999668448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:52512"
ts=2026-04-22T19:21:45.27780205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T19:21:58.590483498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:21:58.592053615Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:22:14.478108185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33486"
ts=2026-04-22T19:22:15.279112615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:22:45.280546047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T19:22:47.816759496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:23:04.003854611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:50688"
ts=2026-04-22T19:23:15.28199499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c92dbcf 10.130.0.69:7946"
ts=2026-04-22T19:23:23.546238323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53618"
level=debug ts=2026-04-22T19:23:30.532834473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:23:45.283283602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:24:14.483446016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51688"
ts=2026-04-22T19:24:15.284550368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T19:24:31.698905792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:24:34.008301882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:40602"
ts=2026-04-22T19:24:45.286392696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T19:25:05.719912276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:25:15.288296987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:25:45.289291447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T19:25:57.814115028Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:25:57.815702234Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:26:15.290397989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:26:34.013757103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47444"
ts=2026-04-22T19:26:45.291823041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T19:26:46.77895389Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:27:00.166413273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53936"
ts=2026-04-22T19:27:00.170485194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53948"
ts=2026-04-22T19:27:05.239898305Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-2c92dbcf (timeout reached)"
ts=2026-04-22T19:27:08.240124462Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-2c92dbcf has failed, no acks received"
ts=2026-04-22T19:27:15.293155045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9a3f9494 10.130.0.69:7946"
ts=2026-04-22T19:27:23.557151906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:33990"
ts=2026-04-22T19:27:28.241029942Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-2c92dbcf as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-22T19:27:40.253397949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:27:45.294784171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:28:04.017910353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:53406"
ts=2026-04-22T19:28:15.296374948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:28:16.078802047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:56908"
level=debug ts=2026-04-22T19:28:32.82927596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:28:34.01890838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58720"
ts=2026-04-22T19:28:45.298165771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:28:46.08074279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58994"
ts=2026-04-22T19:28:53.56103295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59144"
ts=2026-04-22T19:29:04.019775948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58778"
level=debug ts=2026-04-22T19:29:13.242943474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:29:15.299873871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:29:23.561788968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:47748"
level=error ts=2026-04-22T19:29:44.558499273Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:29:44.560622999Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:29:44.56286097Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:29:44.562894053Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:29:45.301469777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T19:29:57.918523146Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:29:57.920718999Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:30:04.023543634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:41836"
ts=2026-04-22T19:30:15.302901259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9a3f9494 10.130.0.69:7946"
ts=2026-04-22T19:30:16.085943271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51500"
ts=2026-04-22T19:30:34.02527395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:36186"
level=debug ts=2026-04-22T19:30:37.669241559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:30:45.304488153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:30:53.566240436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:49210"
ts=2026-04-22T19:31:15.306128988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:31:16.088802718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49374"
level=debug ts=2026-04-22T19:31:20.409999689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:31:34.029270142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:36262"
ts=2026-04-22T19:31:45.307600289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9a3f9494 10.130.0.69:7946"
ts=2026-04-22T19:31:46.090524229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39334"
level=debug ts=2026-04-22T19:32:08.496418917Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:32:15.309143208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:32:16.092284716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59716"
ts=2026-04-22T19:32:34.031839913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:32954"
ts=2026-04-22T19:32:45.310381244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T19:32:47.911743996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:33:15.311721198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:33:16.094822608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33674"
level=debug ts=2026-04-22T19:33:41.847964455Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:33:41.849766769Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:33:45.31304162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:34:04.038299741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:33706"
ts=2026-04-22T19:34:15.314332283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:34:16.097426792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:38382"
level=debug ts=2026-04-22T19:34:16.379820318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:34:20.239759994Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-9a3f9494 (timeout reached)"
ts=2026-04-22T19:34:23.239262302Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-9a3f9494 has failed, no acks received"
ts=2026-04-22T19:34:23.575047091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:45806"
ts=2026-04-22T19:34:31.103126472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:48512"
ts=2026-04-22T19:34:31.107335557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:48520"
ts=2026-04-22T19:34:35.239194247Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-9a3f9494 (timeout reached)"
ts=2026-04-22T19:34:38.23863409Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-9a3f9494 has failed, no acks received"
ts=2026-04-22T19:34:43.239913062Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-9a3f9494 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T19:34:45.239062235Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-9a3f9494 (timeout reached)"
ts=2026-04-22T19:34:45.315231232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:34:48.239167034Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-9a3f9494 has failed, no acks received"
level=debug ts=2026-04-22T19:35:00.274601637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:35:15.316607033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c02061d 10.130.0.69:7946"
ts=2026-04-22T19:35:23.577325016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:38570"
ts=2026-04-22T19:35:45.317804795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T19:35:56.985729225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:36:04.692448285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43746"
ts=2026-04-22T19:36:15.318638524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c02061d 10.130.0.69:7946"
ts=2026-04-22T19:36:23.579660195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:58372"
ts=2026-04-22T19:36:34.693837927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59534"
ts=2026-04-22T19:36:45.321266029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T19:36:49.850030432Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:36:53.581431742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:58876"
ts=2026-04-22T19:37:04.695585308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59178"
ts=2026-04-22T19:37:15.323943291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c02061d 10.130.0.69:7946"
ts=2026-04-22T19:37:23.582834865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:34578"
ts=2026-04-22T19:37:45.326728583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T19:37:51.065113418Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:37:51.06729049Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:37:53.58421073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:48164"
ts=2026-04-22T19:38:04.69856063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43462"
ts=2026-04-22T19:38:15.329083563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c02061d 10.130.0.69:7946"
level=debug ts=2026-04-22T19:38:36.081113126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:38:45.330656599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:38:53.587519466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46054"
ts=2026-04-22T19:39:04.701100227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33938"
level=debug ts=2026-04-22T19:39:14.912608978Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:39:15.341312456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c02061d 10.130.0.69:7946"
level=error ts=2026-04-22T19:39:44.557986489Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:39:44.559955958Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:39:44.561728749Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:39:44.561761372Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:39:45.344725498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c02061d 10.130.0.69:7946"
level=debug ts=2026-04-22T19:39:50.270628699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:40:15.346492704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c02061d 10.130.0.69:7946"
level=debug ts=2026-04-22T19:40:37.161992084Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:40:45.347496934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c02061d 10.130.0.69:7946"
ts=2026-04-22T19:40:53.592208217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:44086"
level=debug ts=2026-04-22T19:41:11.321480127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:41:11.323107614Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:41:15.34899502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c02061d 10.130.0.69:7946"
ts=2026-04-22T19:41:23.594013146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:51530"
ts=2026-04-22T19:41:45.350168452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:42:04.058669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57700"
level=debug ts=2026-04-22T19:42:12.453693995Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:42:15.351221845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c02061d 10.130.0.69:7946"
ts=2026-04-22T19:42:34.060140843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55426"
ts=2026-04-22T19:42:34.709927552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59714"
ts=2026-04-22T19:42:45.353857094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c02061d 10.130.0.69:7946"
level=debug ts=2026-04-22T19:42:56.467916386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:43:04.061625695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:54352"
ts=2026-04-22T19:43:15.355302897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:43:34.062593551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55392"
ts=2026-04-22T19:43:34.713191348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42456"
ts=2026-04-22T19:43:45.356648379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T19:43:49.346215054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:43:53.601376143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:44984"
ts=2026-04-22T19:44:04.063503547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:38918"
ts=2026-04-22T19:44:15.357912337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:44:34.715626469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41612"
level=debug ts=2026-04-22T19:44:43.46978454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:44:45.358708196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c02061d 10.130.0.69:7946"
ts=2026-04-22T19:45:15.360296807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c02061d 10.130.0.69:7946"
level=debug ts=2026-04-22T19:45:19.695194602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:45:19.696694202Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:45:23.604736996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:58662"
ts=2026-04-22T19:45:34.718148001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:35648"
ts=2026-04-22T19:45:45.361939708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T19:46:00.933117721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:46:04.719618099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51850"
ts=2026-04-22T19:46:15.363232899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c02061d 10.130.0.69:7946"
ts=2026-04-22T19:46:23.607213374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:48892"
ts=2026-04-22T19:46:45.364289932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c02061d 10.130.0.69:7946"
ts=2026-04-22T19:46:53.608536775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53668"
level=debug ts=2026-04-22T19:46:59.463240409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:47:04.071337545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:37796"
ts=2026-04-22T19:47:15.365692484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:47:23.610148073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:47424"
ts=2026-04-22T19:47:45.367159263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T19:47:56.381394527Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:48:04.724731413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47786"
ts=2026-04-22T19:48:15.368428102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T19:48:33.725486764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:48:33.860805663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49920"
ts=2026-04-22T19:48:33.865535452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49926"
ts=2026-04-22T19:48:35.238943792Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-2c02061d (timeout reached)"
ts=2026-04-22T19:48:38.239283856Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-2c02061d has failed, no acks received"
ts=2026-04-22T19:48:45.3700137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:48:53.613190089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:58268"
ts=2026-04-22T19:48:58.239431023Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-2c02061d as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T19:49:15.371539563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7dd5f101 10.130.0.69:7946"
level=debug ts=2026-04-22T19:49:17.068868397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:49:17.070645819Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:49:44.558435287Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:49:44.560171055Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:49:44.561800309Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:49:44.56183609Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:49:45.372547091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:49:53.61648786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:36680"
ts=2026-04-22T19:50:07.415593589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43946"
level=debug ts=2026-04-22T19:50:08.626372507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:50:15.374112401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:50:23.617905615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46880"
ts=2026-04-22T19:50:45.374948822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T19:51:07.417607263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34804"
level=debug ts=2026-04-22T19:51:09.215627224Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:51:15.376402451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7dd5f101 10.130.0.69:7946"
ts=2026-04-22T19:51:23.619910522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46152"
ts=2026-04-22T19:51:34.083564714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:34770"
ts=2026-04-22T19:51:45.377798569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7dd5f101 10.130.0.69:7946"
level=debug ts=2026-04-22T19:52:02.463500258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:52:04.084454808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:36154"
ts=2026-04-22T19:52:15.379531205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:52:23.622259783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:39796"
ts=2026-04-22T19:52:34.085769644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:54086"
ts=2026-04-22T19:52:45.381225364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T19:52:56.936635904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:53:04.087381025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:46064"
ts=2026-04-22T19:53:15.382604852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:53:34.088629436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:40522"
level=debug ts=2026-04-22T19:53:41.502319477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:53:41.503889889Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:53:45.383714347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7dd5f101 10.130.0.69:7946"
ts=2026-04-22T19:53:53.626323574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:37838"
ts=2026-04-22T19:54:04.089871263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:51302"
ts=2026-04-22T19:54:07.425198832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34894"
ts=2026-04-22T19:54:15.385318912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7dd5f101 10.130.0.69:7946"
ts=2026-04-22T19:54:37.426429605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:37620"
level=debug ts=2026-04-22T19:54:40.34480021Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:54:45.386766634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:54:53.629638479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:39850"
ts=2026-04-22T19:55:15.388204371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7dd5f101 10.130.0.69:7946"
ts=2026-04-22T19:55:34.094678678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:37034"
level=debug ts=2026-04-22T19:55:38.407468618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:55:45.389701118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:56:15.391129556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T19:56:21.627602374Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:56:37.431018587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46212"
ts=2026-04-22T19:56:45.392172562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T19:56:59.457211895Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:57:15.393564289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:57:37.433693043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42492"
ts=2026-04-22T19:57:45.394492144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T19:57:49.010525464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:57:49.012190451Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:58:04.101448955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:56982"
ts=2026-04-22T19:58:07.43474614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40330"
ts=2026-04-22T19:58:15.39609351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T19:58:33.233352156Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:58:37.436378777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46104"
ts=2026-04-22T19:58:45.397781045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T19:59:07.437657304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:56394"
ts=2026-04-22T19:59:15.398858488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T19:59:28.302773138Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:59:34.106352673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45360"
ts=2026-04-22T19:59:37.438975263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53240"
level=error ts=2026-04-22T19:59:44.558153301Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:59:44.560216413Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:59:44.562029487Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:59:44.562053156Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:59:45.400725694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7dd5f101 10.130.0.69:7946"
ts=2026-04-22T20:00:04.107933236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:51428"
ts=2026-04-22T20:00:07.440669888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36880"
level=debug ts=2026-04-22T20:00:07.718532077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:00:15.401628462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7dd5f101 10.130.0.69:7946"
level=debug ts=2026-04-22T20:00:41.771867347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:00:45.403138282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:00:53.643424768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:57158"
ts=2026-04-22T20:01:15.404474002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:01:23.644958869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:39984"
ts=2026-04-22T20:01:34.113439168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:46364"
level=debug ts=2026-04-22T20:01:36.426049271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:01:36.428146946Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:01:45.405842322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:02:04.114939678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:35866"
ts=2026-04-22T20:02:15.407147581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7dd5f101 10.130.0.69:7946"
ts=2026-04-22T20:02:23.648081137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:60832"
level=debug ts=2026-04-22T20:02:32.730154174Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:02:45.407993006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:03:15.409227594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7dd5f101 10.130.0.69:7946"
level=debug ts=2026-04-22T20:03:18.521613974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:03:23.650292575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:44166"
ts=2026-04-22T20:03:37.451136936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49388"
ts=2026-04-22T20:03:45.410781062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:03:53.651318436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:35566"
ts=2026-04-22T20:04:04.120944685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58922"
ts=2026-04-22T20:04:07.452830576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42376"
level=debug ts=2026-04-22T20:04:10.98103687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:04:15.411645188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:04:23.652289624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:41400"
ts=2026-04-22T20:04:37.454252683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34166"
ts=2026-04-22T20:04:45.413050649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:04:53.653292514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:58768"
ts=2026-04-22T20:05:04.124094898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:48388"
level=debug ts=2026-04-22T20:05:05.639140319Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:05:15.41386487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7dd5f101 10.130.0.69:7946"
ts=2026-04-22T20:05:23.654890912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:48048"
ts=2026-04-22T20:05:45.415306594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T20:05:46.716852529Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:05:46.718462979Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:06:15.417040208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:06:34.128121493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:44204"
level=debug ts=2026-04-22T20:06:42.841971714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:06:45.418785993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:07:04.129585969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57870"
ts=2026-04-22T20:07:07.461793927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:35734"
ts=2026-04-22T20:07:15.420429483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:07:23.659170178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59738"
level=debug ts=2026-04-22T20:07:36.647127077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:07:45.421658029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:08:04.131382579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:35652"
ts=2026-04-22T20:08:07.464161565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:38614"
ts=2026-04-22T20:08:15.423196965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:08:23.661811056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:52290"
level=debug ts=2026-04-22T20:08:25.807645738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:08:45.424804262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:08:53.662867007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53208"
level=debug ts=2026-04-22T20:09:07.375061179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:09:15.426109517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7dd5f101 10.130.0.69:7946"
ts=2026-04-22T20:09:22.345538404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:37414"
ts=2026-04-22T20:09:22.349143418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:37426"
ts=2026-04-22T20:09:23.663979615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:33308"
ts=2026-04-22T20:09:25.240142233Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-7dd5f101 (timeout reached)"
ts=2026-04-22T20:09:28.240244009Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-7dd5f101 has failed, no acks received"
level=error ts=2026-04-22T20:09:44.558565175Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:09:44.560433455Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:09:44.561994528Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:09:44.562021591Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:09:45.427540367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:09:48.240625038Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-7dd5f101 as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-22T20:09:56.509030899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:09:56.51105135Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:10:15.428670576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:10:23.665721093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:50136"
ts=2026-04-22T20:10:45.429920582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T20:10:55.539393317Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:11:15.431703434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-57163e47 10.130.0.69:7946"
level=debug ts=2026-04-22T20:11:33.809991073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:11:34.139860714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45808"
ts=2026-04-22T20:11:45.432995821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-57163e47 10.130.0.69:7946"
ts=2026-04-22T20:12:15.434600966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T20:12:29.953875074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:12:35.670235738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:56322"
ts=2026-04-22T20:12:45.436086724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-57163e47 10.130.0.69:7946"
ts=2026-04-22T20:13:05.671428407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47610"
ts=2026-04-22T20:13:15.437217008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-57163e47 10.130.0.69:7946"
level=debug ts=2026-04-22T20:13:24.423461252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:13:34.145772516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:48900"
ts=2026-04-22T20:13:35.672503627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58082"
ts=2026-04-22T20:13:45.438125235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-57163e47 10.130.0.69:7946"
ts=2026-04-22T20:14:05.673538502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45384"
level=debug ts=2026-04-22T20:14:13.181423282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:14:13.182981621Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:14:15.439155404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:14:23.675377453Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:52132"
ts=2026-04-22T20:14:45.440619452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-57163e47 10.130.0.69:7946"
level=debug ts=2026-04-22T20:15:03.58801996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:15:15.442274506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-57163e47 10.130.0.69:7946"
ts=2026-04-22T20:15:34.150663581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:50106"
ts=2026-04-22T20:15:35.677591813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34080"
ts=2026-04-22T20:15:45.443977861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T20:16:04.364079451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:16:15.44471335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-57163e47 10.130.0.69:7946"
ts=2026-04-22T20:16:35.679572993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39118"
ts=2026-04-22T20:16:45.445801348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:17:04.154987084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:44470"
level=debug ts=2026-04-22T20:17:04.16541113Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:17:15.446943072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:17:34.15598258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:42650"
ts=2026-04-22T20:17:45.447794084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T20:17:46.697262298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:18:05.68281451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49662"
ts=2026-04-22T20:18:15.448694115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-57163e47 10.130.0.69:7946"
level=debug ts=2026-04-22T20:18:22.55840653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:18:22.559833887Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:18:23.685586802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:56002"
ts=2026-04-22T20:18:45.449612758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:19:05.685047177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53310"
level=debug ts=2026-04-22T20:19:09.516945265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:19:15.451264903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=error ts=2026-04-22T20:19:44.557892827Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:19:44.559655565Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:19:44.561265268Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:19:44.561289223Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:19:45.452992829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T20:20:05.506060979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:20:05.687410913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58600"
ts=2026-04-22T20:20:15.454740802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:20:35.689157116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53256"
ts=2026-04-22T20:20:45.455964813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T20:20:50.756550552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:20:53.691286715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:44636"
ts=2026-04-22T20:21:15.457117247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:21:23.692706086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:41322"
level=debug ts=2026-04-22T20:21:36.461318521Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:21:45.458326608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:21:53.693776589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:41360"
ts=2026-04-22T20:22:15.459123036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-57163e47 10.130.0.69:7946"
ts=2026-04-22T20:22:23.694637618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:49672"
ts=2026-04-22T20:22:34.170186851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:49440"
ts=2026-04-22T20:22:35.695161888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:38334"
level=debug ts=2026-04-22T20:22:36.177037291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:22:36.178616534Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:22:45.460159909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T20:23:13.178268544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:23:15.461184708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:23:23.697644647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:42642"
ts=2026-04-22T20:23:45.462525713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T20:23:57.533618331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:24:15.464251422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-57163e47 10.130.0.69:7946"
ts=2026-04-22T20:24:23.699583675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:37330"
ts=2026-04-22T20:24:38.88598839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40788"
ts=2026-04-22T20:24:38.88974582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40796"
level=debug ts=2026-04-22T20:24:44.682980286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:24:45.23963108Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-57163e47 (timeout reached)"
ts=2026-04-22T20:24:45.465305831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:24:48.238746473Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-57163e47 has failed, no acks received"
ts=2026-04-22T20:25:03.456578487Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-57163e47 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T20:25:05.239143407Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-57163e47 (timeout reached)"
ts=2026-04-22T20:25:08.239035119Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-57163e47 has failed, no acks received"
ts=2026-04-22T20:25:15.466604615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T20:25:20.069402635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:25:45.468183443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:25:53.702466077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:51568"
level=debug ts=2026-04-22T20:26:12.9917103Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:26:12.993205239Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:26:15.469775118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:26:23.7040966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:44748"
ts=2026-04-22T20:26:29.048950672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:57454"
ts=2026-04-22T20:26:45.470971373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:26:53.704844597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:43976"
level=debug ts=2026-04-22T20:26:57.665658511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:27:04.182266437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:34956"
ts=2026-04-22T20:27:15.472562465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:27:34.18361529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57350"
level=debug ts=2026-04-22T20:27:37.288441279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:27:45.473291957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d8d96911 10.130.0.69:7946"
ts=2026-04-22T20:27:53.706872099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:37556"
ts=2026-04-22T20:28:04.184387622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:38152"
ts=2026-04-22T20:28:15.474326813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:28:23.708299726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:35172"
level=debug ts=2026-04-22T20:28:27.253745328Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:28:34.185706015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:41648"
ts=2026-04-22T20:28:45.475187994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d8d96911 10.130.0.69:7946"
ts=2026-04-22T20:29:04.187527146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:50834"
ts=2026-04-22T20:29:15.476786067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d8d96911 10.130.0.69:7946"
level=debug ts=2026-04-22T20:29:19.526304351Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:29:29.057525187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52270"
level=error ts=2026-04-22T20:29:44.558276027Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:29:44.560035369Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:29:44.561756995Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:29:44.561782382Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:29:45.478174681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:29:59.058426863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47492"
level=debug ts=2026-04-22T20:30:02.152535957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:30:02.154230624Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:30:15.479535177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:30:45.480887086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T20:30:50.145178638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:30:53.714555823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:38038"
ts=2026-04-22T20:31:15.481664503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:31:23.715138897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:38158"
ts=2026-04-22T20:31:45.48256347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d8d96911 10.130.0.69:7946"
level=debug ts=2026-04-22T20:31:45.610853159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:32:04.195347874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:43794"
ts=2026-04-22T20:32:15.48404585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:32:23.716777499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55856"
level=debug ts=2026-04-22T20:32:31.317915951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:32:34.196447595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:56840"
ts=2026-04-22T20:32:45.485777147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:33:15.487401158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T20:33:24.547761038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:33:29.068819599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:32918"
ts=2026-04-22T20:33:34.199151997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:42864"
ts=2026-04-22T20:33:45.488716717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
ts=2026-04-22T20:34:15.489848753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
level=debug ts=2026-04-22T20:34:22.476622409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:34:22.478173126Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:34:23.721534104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:50684"
ts=2026-04-22T20:34:45.491289405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:34:53.722399019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:50512"
ts=2026-04-22T20:35:04.203705264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:50186"
ts=2026-04-22T20:35:06.748466669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33242"
ts=2026-04-22T20:35:06.75249509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33252"
ts=2026-04-22T20:35:15.49223358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f12c83f 10.130.0.69:7946"
level=debug ts=2026-04-22T20:35:16.890132851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:35:20.239715601Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-d8d96911 (timeout reached)"
ts=2026-04-22T20:35:23.239263428Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-d8d96911 has failed, no acks received"
ts=2026-04-22T20:35:23.723394004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:38624"
ts=2026-04-22T20:35:40.238996489Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-d8d96911 (timeout reached)"
ts=2026-04-22T20:35:43.239008826Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-d8d96911 has failed, no acks received"
ts=2026-04-22T20:35:45.493785899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"
level=debug ts=2026-04-22T20:36:00.770841052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:36:06.647285419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53404"
ts=2026-04-22T20:36:15.494891003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cmnr6-8ae9f4a2 10.130.0.61:7946"
ts=2026-04-22T20:36:23.726223518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:35168"
ts=2026-04-22T20:36:34.207452682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:48470"
ts=2026-04-22T20:36:45.495712342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6f12c83f 10.130.0.69:7946"
level=debug ts=2026-04-22T20:36:56.868236206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:37:04.208630007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:39350"
ts=2026-04-22T20:37:15.49652921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rcs8h-79373732 10.130.0.65:7946"