level=info ts=2026-05-05T22:46:03.348604546Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-05T22:46:03.348651163Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-05T22:46:03.349617179Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-05T22:46:03.349946517Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-05T22:46:03.350020656Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.63 inf=eth0
level=debug ts=2026-05-05T22:46:03.351387112Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-05T22:46:03.351503892Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.63 inf=eth0
level=info ts=2026-05-05T22:46:03.351530936Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-452fa925
ts=2026-05-05T22:46:03.351644532Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-05T22:46:03.351662836Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.63 advertisePort=7946
level=debug ts=2026-05-05T22:46:03.351680838Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.63 advertisePort=7946
level=info ts=2026-05-05T22:46:03.351828054Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-05T22:46:03.358520961Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T22:46:03.358547719Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-05T22:46:03.358561685Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.736526ms
level=error ts=2026-05-05T22:46:03.358573492Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-05T22:46:03.358589847Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-05T22:46:03.365477441Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T22:46:03.365514664Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-05T22:46:03.368843318Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-05T22:46:03.368874243Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-05T22:46:03.369565085Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-05T22:46:03.369979344Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-05-05T22:46:03.370083303Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-05-05T22:46:03.370091469Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-05T22:46:03.370147158Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-05-05T22:46:03.370153834Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-05T22:46:03.370452283Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-05T22:46:03.370460117Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-05T22:46:03.370519324Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-05T22:46:03.370562925Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-05T22:46:03.370586916Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-05T22:46:03.370733178Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-05T22:46:03.37074286Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-05T22:46:03.370757237Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-05T22:46:04.37128574Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-05T22:46:04.495323502Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-05T22:46:04.4953961Z caller=loki.go:615 msg="Loki started" startup_time=1.218783546s
ts=2026-05-05T22:46:04.511062086Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T22:46:04.511083268Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T22:46:06.841369289Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T22:46:06.841395228Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-05T22:46:09.495829962Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-05T22:46:09.495882946Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-05T22:46:14.646164673Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T22:46:14.64619151Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-05T22:46:22.083827638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T22:46:26.673285601Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.78:7946: dial tcp 10.129.0.78:7946: connect: connection refused"
ts=2026-05-05T22:46:26.674512213Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.80:7946: dial tcp 10.129.0.80:7946: connect: connection refused"
level=warn ts=2026-05-05T22:46:26.674521637Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="2 errors occurred:\n\t* failed to join 10.129.0.78:7946: dial tcp 10.129.0.78:7946: connect: connection refused\n\t* failed to join 10.129.0.80:7946: dial tcp 10.129.0.80:7946: connect: connection refused\n\n"
ts=2026-05-05T22:46:36.168020284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:41572"
level=debug ts=2026-05-05T22:46:44.257503113Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T22:46:46.099505962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:60554"
ts=2026-05-05T22:46:48.320268872Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.78:7946: dial tcp 10.129.0.78:7946: connect: connection refused"
ts=2026-05-05T22:46:48.321269549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.79:7946"
ts=2026-05-05T22:46:48.323825431Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.80:7946: dial tcp 10.129.0.80:7946: connect: connection refused"
ts=2026-05-05T22:46:48.323920834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.63:7946"
ts=2026-05-05T22:46:48.323959424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:45768"
ts=2026-05-05T22:46:48.324533197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.61:7946"
ts=2026-05-05T22:46:48.32509605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.59:7946"
ts=2026-05-05T22:46:48.326455047Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.62:7946: dial tcp 10.130.0.62:7946: connect: connection refused"
level=info ts=2026-05-05T22:46:48.326465797Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=44.967868883s
ts=2026-05-05T22:46:51.387502983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T22:46:55.843145253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55546"
level=debug ts=2026-05-05T22:47:05.766715042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T22:47:06.217987539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:49128"
ts=2026-05-05T22:47:21.389373762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-05T22:47:28.758335264Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T22:47:36.219153676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:33278"
ts=2026-05-05T22:47:42.004668491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:42032"
ts=2026-05-05T22:47:51.390907653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-05T22:47:59.138702873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-05T22:47:59.140291986Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T22:48:21.392263177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T22:48:24.925245339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:49914"
ts=2026-05-05T22:48:36.22469784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:39198"
level=debug ts=2026-05-05T22:48:46.744756104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T22:48:51.394234681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T22:49:21.396476876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T22:49:51.397698917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-05T22:50:03.088522122Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T22:50:12.011196469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58680"
ts=2026-05-05T22:50:21.399047312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T22:50:42.012861918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:38182"
ts=2026-05-05T22:50:51.401379387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T22:50:54.940272619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:56928"
ts=2026-05-05T22:51:12.014440834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:56970"
level=debug ts=2026-05-05T22:51:14.279545678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T22:51:21.403867934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T22:51:51.405280388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-05T22:52:17.665142548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T22:52:21.406604751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T22:52:51.407703457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-05T22:53:17.56172625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-05T22:53:17.564265809Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T22:53:21.410002153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T22:53:24.954841917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:36634"
ts=2026-05-05T22:53:51.412048175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T22:54:06.245684622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:42314"
level=debug ts=2026-05-05T22:54:07.034212649Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T22:54:21.414048382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T22:54:51.41603692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T22:54:54.962674278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:53654"
level=debug ts=2026-05-05T22:55:06.044800101Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T22:55:21.417310747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T22:55:42.026873975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:59942"
level=debug ts=2026-05-05T22:55:49.640020981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T22:55:51.418414639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T22:56:06.254991044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:59502"
level=info ts=2026-05-05T22:56:09.496060835Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-05T22:56:09.499590414Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T22:56:09.501428056Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T22:56:09.503149167Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T22:56:09.503173667Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T22:56:09.503186001Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-05T22:56:21.420380953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T22:56:24.971148555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:41624"
level=debug ts=2026-05-05T22:56:27.964314136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T22:56:51.422407823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T22:57:06.260263894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:55942"
ts=2026-05-05T22:57:21.424189169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-05T22:57:24.713502074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T22:57:24.715209651Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T22:57:51.425050028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T22:58:06.263887261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:53316"
ts=2026-05-05T22:58:12.036814032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55956"
ts=2026-05-05T22:58:21.427537653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T22:58:22.696830646Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T22:58:36.265084381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:53466"
ts=2026-05-05T22:58:42.03779907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:44804"
ts=2026-05-05T22:58:51.429874818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T22:59:06.266559668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:44028"
ts=2026-05-05T22:59:12.038836629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:36846"
level=debug ts=2026-05-05T22:59:15.996160587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T22:59:21.431695929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T22:59:24.98820291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:32946"
ts=2026-05-05T22:59:51.433176094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:00:06.269918662Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:53080"
level=debug ts=2026-05-05T23:00:15.281244328Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:00:21.435179576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:00:36.271526082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:43616"
ts=2026-05-05T23:00:51.43817055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:00:54.99621446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:43282"
level=debug ts=2026-05-05T23:01:00.744372851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:01:21.441680747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:01:41.67129037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T23:01:41.673455325Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:01:42.048172931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:39418"
ts=2026-05-05T23:01:51.444103893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:01:55.001819403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:55612"
ts=2026-05-05T23:02:12.049560702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:36460"
ts=2026-05-05T23:02:21.445933688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:02:25.691829179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:02:51.447696277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-05T23:03:03.411425895Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:03:06.282797197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:53262"
ts=2026-05-05T23:03:21.448824076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-05T23:03:50.466936485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:03:51.450150596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:04:21.452288214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:04:27.814917393Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:04:36.290634029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:33724"
ts=2026-05-05T23:04:51.454983117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:05:21.45697988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:05:22.731548638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T23:05:22.733406436Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:05:25.023611804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:38962"
ts=2026-05-05T23:05:42.066442749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:37100"
ts=2026-05-05T23:05:51.459590959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:06:06.29813445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:34276"
level=error ts=2026-05-05T23:06:09.507535528Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:06:09.509684932Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:06:09.511468979Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:06:09.511494042Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:06:12.067543767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:35170"
level=debug ts=2026-05-05T23:06:20.667685228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:06:21.462247333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:06:51.464259433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:06:55.033512666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:55410"
ts=2026-05-05T23:07:06.303019104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:37302"
level=debug ts=2026-05-05T23:07:12.042063716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:07:12.07046128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:40640"
ts=2026-05-05T23:07:21.466332617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:07:51.467791505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-05T23:07:51.982782955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:08:06.306204494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:52292"
ts=2026-05-05T23:08:21.46896979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-05T23:08:30.84521518Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:08:51.47063288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:08:55.04514309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:37210"
ts=2026-05-05T23:09:21.472560973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-05T23:09:30.972766741Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T23:09:30.974549784Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:09:42.081083216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:33300"
ts=2026-05-05T23:09:51.474172541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:10:21.476913573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:10:26.569305592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:10:36.316255895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:35244"
ts=2026-05-05T23:10:51.479109717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-05T23:11:04.869731474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:11:21.480249013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-05T23:11:47.881783399Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:11:51.482119904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:12:21.485182404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:12:36.324942252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:57230"
level=debug ts=2026-05-05T23:12:40.030285377Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:12:51.487569378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:12:55.070870574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:60176"
ts=2026-05-05T23:13:21.490099285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:13:25.073494627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:51264"
level=debug ts=2026-05-05T23:13:40.460409598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T23:13:40.46263584Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:13:51.491782944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:14:21.493100626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-05T23:14:36.079648252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:14:51.495433692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:15:06.338791703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:42544"
level=debug ts=2026-05-05T23:15:20.081143398Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:15:21.497027101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:15:25.083270035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:55854"
ts=2026-05-05T23:15:51.498701342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-05T23:16:02.774167766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T23:16:09.506370427Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:16:09.508133797Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:16:09.509724312Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:16:09.509749148Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:16:21.500452627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:16:51.502830862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:16:55.091162962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:58664"
level=debug ts=2026-05-05T23:16:55.718933272Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:17:06.345935428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:42974"
ts=2026-05-05T23:17:12.108190276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55202"
ts=2026-05-05T23:17:21.505555801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:17:25.093604604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:33090"
ts=2026-05-05T23:17:36.347267466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:42900"
ts=2026-05-05T23:17:42.110095599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:54052"
level=debug ts=2026-05-05T23:17:42.511182894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T23:17:42.513983519Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:17:51.508098735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:17:55.096905389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:45526"
ts=2026-05-05T23:18:06.348720293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:58698"
ts=2026-05-05T23:18:21.510117633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:18:40.097487477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:18:51.512174035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:18:55.103084325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:43566"
ts=2026-05-05T23:19:21.517696376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:19:23.890551264Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:19:51.519948163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:19:55.108719452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:46240"
level=debug ts=2026-05-05T23:20:18.961286744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:20:21.522952697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:20:42.124733957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58736"
ts=2026-05-05T23:20:51.525233984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:20:55.115026173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:57904"
level=debug ts=2026-05-05T23:20:59.523279376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:21:21.527256229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-05T23:21:48.42571099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T23:21:48.431690139Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:21:51.52855878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:21:55.122187917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:32894"
ts=2026-05-05T23:22:06.364604108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:55360"
ts=2026-05-05T23:22:12.13119701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:33022"
ts=2026-05-05T23:22:21.529715986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-05T23:22:31.830880874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:22:36.36594264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:56148"
ts=2026-05-05T23:22:51.530766856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:22:55.128257299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:59320"
ts=2026-05-05T23:23:06.366918147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:42160"
ts=2026-05-05T23:23:21.532783577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:23:21.780105209Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:23:42.136280172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:43604"
ts=2026-05-05T23:23:51.534594324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:24:12.137390371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58058"
level=debug ts=2026-05-05T23:24:13.715844114Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:24:21.535879651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:24:36.371363988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:38132"
ts=2026-05-05T23:24:42.139076301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:34052"
ts=2026-05-05T23:24:51.537153847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-05T23:24:59.417089942Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:25:06.372718066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:53672"
ts=2026-05-05T23:25:21.538998413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:25:36.374716936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:43874"
ts=2026-05-05T23:25:42.142852918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:37376"
level=debug ts=2026-05-05T23:25:44.632861032Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T23:25:44.634532602Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:25:51.541333257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:25:55.146101928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:43002"
ts=2026-05-05T23:26:06.375635341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:34450"
level=error ts=2026-05-05T23:26:09.50739059Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:26:09.509430495Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:26:09.511419241Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:26:09.511452488Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:26:12.143948877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:33932"
ts=2026-05-05T23:26:21.54398504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:26:26.191528164Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:26:42.145798294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58230"
ts=2026-05-05T23:26:51.546215437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:27:06.378132005Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:37412"
ts=2026-05-05T23:27:12.147048425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55242"
level=debug ts=2026-05-05T23:27:19.209862717Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:27:21.547769594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:27:42.148182884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:49420"
ts=2026-05-05T23:27:51.548855886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:27:55.158213258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:60156"
ts=2026-05-05T23:28:12.150004976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:38216"
level=debug ts=2026-05-05T23:28:18.576305577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:28:21.551192604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:28:42.151584636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:41248"
ts=2026-05-05T23:28:51.556540665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:28:55.164003055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:35576"
ts=2026-05-05T23:29:12.152515084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:33228"
level=debug ts=2026-05-05T23:29:13.7698569Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:29:21.558058731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:29:25.166632886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:41514"
ts=2026-05-05T23:29:51.559702484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-05T23:29:58.615170495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T23:29:58.61682109Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:30:12.156655574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:52988"
ts=2026-05-05T23:30:21.562321068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:30:25.171911923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:45196"
level=debug ts=2026-05-05T23:30:45.99985139Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:30:51.564335307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:30:55.175038397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:58634"
ts=2026-05-05T23:31:12.161720985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:39274"
ts=2026-05-05T23:31:21.566682073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:31:25.178320369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:41020"
level=debug ts=2026-05-05T23:31:40.333234848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:31:42.163316934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:60380"
ts=2026-05-05T23:31:51.568359908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:32:06.401306054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:54412"
ts=2026-05-05T23:32:12.165256395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57658"
ts=2026-05-05T23:32:21.570837626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:32:40.249162999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:32:51.573575414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:33:06.405533118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:43380"
ts=2026-05-05T23:33:21.576985754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:33:34.665106494Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:33:51.57929099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:33:55.193358258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:36154"
ts=2026-05-05T23:34:21.581072231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:34:23.595188901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T23:34:23.597085807Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:34:51.58281979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:35:06.414402787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:53980"
ts=2026-05-05T23:35:12.180447971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:48928"
ts=2026-05-05T23:35:21.58363275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-05T23:35:22.191969004Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:35:51.585508366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-05T23:36:00.398254944Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T23:36:09.50642991Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:36:09.508561527Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:36:09.51018263Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:36:09.510209313Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:36:21.587867245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:36:44.214988728Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:36:51.590599234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:37:12.188030731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:46684"
ts=2026-05-05T23:37:21.59351469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:37:25.214771825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:33930"
ts=2026-05-05T23:37:42.18894805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57544"
level=debug ts=2026-05-05T23:37:43.070037114Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:37:51.596025336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:37:55.218066032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:58902"
ts=2026-05-05T23:38:06.430947765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:37046"
ts=2026-05-05T23:38:21.599425474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:38:37.665113645Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T23:38:37.66681151Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:38:42.192444592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:33404"
ts=2026-05-05T23:38:51.602801639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:39:19.712139759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:39:21.606029215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:39:36.436057474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:41820"
ts=2026-05-05T23:39:51.60833286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-05T23:39:56.454668519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:40:06.437628981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:55690"
ts=2026-05-05T23:40:21.610132734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:40:36.438627286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:45178"
level=debug ts=2026-05-05T23:40:39.513198053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:40:51.611939509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:40:55.236742277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:57576"
ts=2026-05-05T23:41:12.200770436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:41474"
level=debug ts=2026-05-05T23:41:14.551227427Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:41:21.614164624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:41:42.201780397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58640"
ts=2026-05-05T23:41:51.615574592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:41:55.242895646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:50942"
level=debug ts=2026-05-05T23:41:57.428515716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T23:41:57.429991096Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:42:06.445076236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:42346"
ts=2026-05-05T23:42:21.618209435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:42:36.446216735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:55542"
ts=2026-05-05T23:42:51.621421075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:42:51.815059025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:43:12.207505896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:56602"
ts=2026-05-05T23:43:21.624446599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:43:39.089242985Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:43:51.626800468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:44:06.451846516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:59808"
level=debug ts=2026-05-05T23:44:17.592753385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:44:21.627909415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:44:25.257683524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:40186"
ts=2026-05-05T23:44:51.629294874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:44:55.26041769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:38350"
level=debug ts=2026-05-05T23:45:08.840829436Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:45:21.630647527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:45:25.263731447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:42210"
ts=2026-05-05T23:45:36.457846455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:38648"
ts=2026-05-05T23:45:51.632325845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:46:03.386570638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T23:46:03.388071653Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:46:09.505182179Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:46:09.506832935Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:46:09.508360645Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:46:09.508386174Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:46:21.63450937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:46:42.221943784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:37344"
level=debug ts=2026-05-05T23:46:49.521979185Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:46:51.636433582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:46:55.272919927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:48066"
ts=2026-05-05T23:47:21.6375457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:47:25.276199073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:49756"
ts=2026-05-05T23:47:51.639903039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:47:51.695257724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:48:12.229488604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:49600"
ts=2026-05-05T23:48:21.643771864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:48:36.471616163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:60592"
level=debug ts=2026-05-05T23:48:44.548845434Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:48:51.645637991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:48:55.285593786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:52214"
ts=2026-05-05T23:49:06.473232691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:44942"
ts=2026-05-05T23:49:21.647654231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:49:36.474283344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:51854"
level=debug ts=2026-05-05T23:49:37.002131284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:49:51.650331389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:49:55.290685737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:48480"
ts=2026-05-05T23:50:06.475541187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:54048"
ts=2026-05-05T23:50:21.652270728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-05T23:50:23.588834733Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T23:50:23.590381068Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:50:25.293496475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:55310"
ts=2026-05-05T23:50:36.477438621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:59016"
ts=2026-05-05T23:50:51.654584994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:50:55.29612201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:34594"
ts=2026-05-05T23:51:21.657488924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-05T23:51:22.216338578Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:51:36.479856246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:51830"
ts=2026-05-05T23:51:42.242272671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45204"
ts=2026-05-05T23:51:51.660114294Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:51:55.30208744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:34618"
level=debug ts=2026-05-05T23:52:12.976257472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:52:21.662319327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:52:25.30458225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:48840"
level=debug ts=2026-05-05T23:52:49.707121343Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:52:51.664887826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:52:55.306865012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:40596"
ts=2026-05-05T23:53:21.668198928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-05T23:53:32.550305554Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:53:42.251847182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:52392"
ts=2026-05-05T23:53:51.670126306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:53:55.312679411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:39772"
ts=2026-05-05T23:54:12.252846429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:35786"
level=debug ts=2026-05-05T23:54:20.051477294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T23:54:20.053436491Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:54:21.671305011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-05T23:54:42.254503121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47280"
ts=2026-05-05T23:54:51.67255849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-05T23:54:56.008688912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:55:12.256368757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:44842"
ts=2026-05-05T23:55:21.673612604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-05T23:55:51.175224686Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:55:51.67475065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:56:06.502845224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:35916"
level=error ts=2026-05-05T23:56:09.508492059Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:56:09.510259272Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:56:09.511814851Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T23:56:09.511835565Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:56:21.675733582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:56:25.326200354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:52628"
ts=2026-05-05T23:56:36.503798494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:59114"
level=debug ts=2026-05-05T23:56:48.906842576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:56:51.67705472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:56:55.329299667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:40098"
ts=2026-05-05T23:57:21.677815249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-05T23:57:31.923395219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:57:51.679124461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-05T23:57:55.3353132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:40152"
ts=2026-05-05T23:58:21.680978634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-05T23:58:21.935018926Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T23:58:21.936689666Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T23:58:42.268859598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:37394"
ts=2026-05-05T23:58:51.682551448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-05T23:58:58.661989243Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:59:21.685174698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-05T23:59:25.343263793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:53902"
ts=2026-05-05T23:59:36.513326018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:36512"
level=debug ts=2026-05-05T23:59:43.796451511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T23:59:51.688279046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-06T00:00:21.691639188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-06T00:00:23.814368135Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:00:25.349194967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:39594"
ts=2026-05-06T00:00:51.694254345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-06T00:01:20.334687825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:01:21.696752333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-06T00:01:51.697667427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-06T00:01:57.895642663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T00:01:57.897986582Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T00:02:21.699547835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-06T00:02:51.70136684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-06T00:02:53.39075051Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:03:12.285849155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:40858"
ts=2026-05-06T00:03:21.702563359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-06T00:03:36.528507702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:53178"
level=debug ts=2026-05-06T00:03:40.507922352Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:03:42.286938087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:41202"
ts=2026-05-06T00:03:51.703893534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-06T00:04:12.288686003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58298"
ts=2026-05-06T00:04:21.705090665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-06T00:04:22.54197426Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:04:51.706842379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-06T00:05:06.534850779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:35942"
level=debug ts=2026-05-06T00:05:06.840268886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:05:21.708447639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-06T00:05:42.294266714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:34610"
ts=2026-05-06T00:05:51.709895746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-06T00:05:55.380678222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:50880"
ts=2026-05-06T00:06:06.538745664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:56492"
level=debug ts=2026-05-06T00:06:08.186474746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T00:06:08.18822992Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T00:06:09.505567388Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T00:06:09.507086521Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T00:06:09.50851851Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T00:06:09.508542654Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T00:06:21.712325509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-06T00:06:36.540096446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:35836"
ts=2026-05-06T00:06:51.714286841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-06T00:07:02.626615675Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:07:21.716685601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-06T00:07:25.388884021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:45748"
ts=2026-05-06T00:07:36.543761513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:37492"
ts=2026-05-06T00:07:42.302402535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:42592"
ts=2026-05-06T00:07:51.719105225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-06T00:08:02.053220752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:08:21.721727487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-06T00:08:25.394546858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:44588"
ts=2026-05-06T00:08:36.548204092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:52412"
ts=2026-05-06T00:08:51.724050967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-06T00:08:55.680749708Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:09:21.725952749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-06T00:09:25.39973493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:52348"
level=debug ts=2026-05-06T00:09:33.273153374Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:09:42.309276436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55866"
ts=2026-05-06T00:09:51.727538064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-06T00:10:12.310833341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:39954"
ts=2026-05-06T00:10:21.729178825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-06T00:10:25.405624509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:52308"
level=debug ts=2026-05-06T00:10:33.651434324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T00:10:33.653441006Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T00:10:51.731164258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-06T00:10:55.408834316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:49992"
ts=2026-05-06T00:11:06.556896418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:52538"
ts=2026-05-06T00:11:12.314112336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55516"
level=debug ts=2026-05-06T00:11:13.770086386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:11:21.733803067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-06T00:11:36.557997081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:54866"
ts=2026-05-06T00:11:51.735946583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-06T00:11:58.257310871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:12:21.737774322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-06T00:12:47.716480814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:12:51.740436873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-06T00:13:06.56540229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:43112"
ts=2026-05-06T00:13:21.74221686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-06T00:13:49.815766314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:13:51.743779324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-06T00:14:21.745542422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-06T00:14:36.571969873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:37452"
level=debug ts=2026-05-06T00:14:42.026792279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T00:14:42.029178426Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T00:14:51.747335175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-06T00:15:06.573636303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:43952"
ts=2026-05-06T00:15:21.749490089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-06T00:15:25.435195831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:43818"
ts=2026-05-06T00:15:36.575293431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:58620"
level=debug ts=2026-05-06T00:15:40.715675232Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:15:42.333982062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:51246"
ts=2026-05-06T00:15:51.751760398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-06T00:16:06.576155969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:51866"
level=error ts=2026-05-06T00:16:09.507617339Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T00:16:09.509355178Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T00:16:09.510960688Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T00:16:09.510982733Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T00:16:12.335567133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55672"
level=debug ts=2026-05-06T00:16:20.763446447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:16:21.753096469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-06T00:16:36.577776268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:43766"
ts=2026-05-06T00:16:51.755669996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-06T00:16:55.443147903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:54108"
level=debug ts=2026-05-06T00:17:01.286267636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:17:06.579634536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:46468"
ts=2026-05-06T00:17:12.3378611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45116"
ts=2026-05-06T00:17:21.758009505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-06T00:17:25.445833196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:48962"
ts=2026-05-06T00:17:42.339123717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45990"
ts=2026-05-06T00:17:51.759243576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-06T00:17:58.322403293Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:18:21.76097322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-06T00:18:25.451495303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:42996"
ts=2026-05-06T00:18:36.584149397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:43566"
level=debug ts=2026-05-06T00:18:50.339774191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T00:18:50.341549964Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T00:18:51.762975682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-06T00:18:55.454170021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:56568"
ts=2026-05-06T00:19:21.765608047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-06T00:19:35.743231816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:19:51.768913416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-06T00:19:55.460029059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:37386"
ts=2026-05-06T00:20:06.588503522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:35184"
ts=2026-05-06T00:20:21.771259694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-06T00:20:26.845399586Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:20:51.772675442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-06T00:21:21.773695178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-06T00:21:22.624649212Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:21:25.468565843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:40714"
ts=2026-05-06T00:21:51.775488262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-06T00:21:55.471434417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:60368"
ts=2026-05-06T00:22:12.356572111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:40658"
level=debug ts=2026-05-06T00:22:18.070581654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:22:21.777907779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-06T00:22:51.779307031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-06T00:23:01.455884449Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T00:23:01.457474257Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T00:23:21.780812777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
level=debug ts=2026-05-06T00:23:41.258807776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:23:51.782418543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-06T00:24:21.784654917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
level=debug ts=2026-05-06T00:24:37.151437117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:24:51.787994853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-06T00:25:06.607848937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:37418"
level=debug ts=2026-05-06T00:25:16.990979231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:25:21.790643594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2984e69 10.129.0.79:7946"
ts=2026-05-06T00:25:36.609533452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:45916"
ts=2026-05-06T00:25:51.792131636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"
ts=2026-05-06T00:25:55.493685605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:46548"
level=debug ts=2026-05-06T00:26:06.321998878Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T00:26:06.610797071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:53824"
level=error ts=2026-05-06T00:26:09.506272058Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T00:26:09.507777418Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T00:26:09.509150993Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T00:26:09.509171733Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T00:26:21.79380631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
level=debug ts=2026-05-06T00:26:51.47149903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T00:26:51.473103685Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T00:26:51.795305813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2bklh-ca27f793 10.130.0.61:7946"
ts=2026-05-06T00:27:06.61584255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:49820"
ts=2026-05-06T00:27:21.796246585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-lz979-bba8db1c 10.130.0.59:7946"