level=info ts=2026-05-05T11:25:15.126468344Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-05T11:25:15.126497619Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-05T11:25:15.128020251Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-05T11:25:15.128377861Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-05T11:25:15.128449934Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.63 inf=eth0
level=debug ts=2026-05-05T11:25:15.129369418Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-05T11:25:15.129444616Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.63 inf=eth0
level=info ts=2026-05-05T11:25:15.129642534Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-49342479
ts=2026-05-05T11:25:15.129706241Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-05T11:25:15.129717577Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.63 advertisePort=7946
level=debug ts=2026-05-05T11:25:15.129731788Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.63 advertisePort=7946
level=info ts=2026-05-05T11:25:15.129839748Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-05T11:25:15.136996196Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T11:25:15.137008434Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-05T11:25:15.137027938Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=7.190295ms
level=error ts=2026-05-05T11:25:15.137038978Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-05T11:25:15.137051724Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-05T11:25:15.13997942Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T11:25:15.139991167Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-05T11:25:15.144446823Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-05T11:25:15.144459062Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-05-05T11:25:15.144486624Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-05T11:25:15.144480874Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-05T11:25:15.144491391Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-05-05T11:25:15.14449338Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-05T11:25:15.144567089Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-05T11:25:15.144571502Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-05T11:25:15.144783498Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-05T11:25:15.144798938Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-05T11:25:15.14487023Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-05T11:25:15.144862113Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-05T11:25:15.144890543Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-05T11:25:15.145086242Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-05T11:25:15.286361596Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-05T11:25:15.286407031Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-05T11:25:16.287382042Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-05T11:25:16.466914201Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-05T11:25:16.466953595Z caller=loki.go:615 msg="Loki started" startup_time=1.401005162s
ts=2026-05-05T11:25:17.113125334Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T11:25:17.113147729Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:25:21.029749519Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T11:25:21.02977074Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-05T11:25:21.467787528Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-05T11:25:21.467837676Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-05T11:25:27.979336122Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T11:25:27.979360783Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-05T11:25:34.992149998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:25:37.7875605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:54496"
ts=2026-05-05T11:25:38.230528435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41300"
ts=2026-05-05T11:25:38.729381639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.63:7946"
ts=2026-05-05T11:25:38.729484365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:32778"
ts=2026-05-05T11:25:38.730816791Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.64:7946: dial tcp 10.130.0.64:7946: connect: connection refused"
level=info ts=2026-05-05T11:25:38.730826744Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=23.593769086s
ts=2026-05-05T11:25:40.807563861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56800"
level=debug ts=2026-05-05T11:25:51.699602262Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:25:55.926930302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:26:14.591939082Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:26:17.984806522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:53586"
ts=2026-05-05T11:26:18.852777489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:52282"
ts=2026-05-05T11:26:25.929148371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T11:26:40.686829127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:26:48.855562395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:38540"
ts=2026-05-05T11:26:55.931830962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T11:26:58.877529666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51456"
level=debug ts=2026-05-05T11:27:13.659317311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-05T11:27:13.661140176Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:27:18.857448112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:44238"
ts=2026-05-05T11:27:25.934185548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T11:27:47.989835755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:36278"
ts=2026-05-05T11:27:48.859648917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:40970"
ts=2026-05-05T11:27:55.936366622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:27:56.592858183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:28:25.938531836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T11:28:48.864805547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:55862"
ts=2026-05-05T11:28:55.941471554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:29:05.828018621Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:29:18.866984512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:40662"
ts=2026-05-05T11:29:25.944402052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:29:55.668085608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:29:55.946924394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T11:29:58.897542627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48996"
ts=2026-05-05T11:30:18.870863659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:43950"
ts=2026-05-05T11:30:25.949707922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T11:30:48.00460849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:59474"
ts=2026-05-05T11:30:55.951630239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:31:13.803519242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:31:25.954423515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T11:31:28.907705075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35258"
ts=2026-05-05T11:31:48.008014915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:59818"
ts=2026-05-05T11:31:55.957111854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:32:20.148455793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-05T11:32:20.15022319Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:32:25.959497693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T11:32:55.961592143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T11:33:15.683226775Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:33:18.014251447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:38484"
ts=2026-05-05T11:33:25.96365024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T11:33:28.919499499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42374"
ts=2026-05-05T11:33:48.887539714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:53460"
ts=2026-05-05T11:33:55.966615204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T11:34:11.799717462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:34:25.969561212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T11:34:55.972394413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:34:57.802931171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:35:18.894586251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:37390"
level=info ts=2026-05-05T11:35:21.468674129Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-05T11:35:21.476039967Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:35:21.47795972Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:35:21.479771074Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:35:21.479793209Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T11:35:21.479807867Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-05T11:35:25.974882247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T11:35:28.931419422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46002"
ts=2026-05-05T11:35:55.977643517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:35:56.82890466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:35:58.933823966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:54378"
ts=2026-05-05T11:36:25.979655154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T11:36:55.981666749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:36:58.797256656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:36:58.799341886Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:37:18.906260695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:38304"
ts=2026-05-05T11:37:25.984357031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:37:45.282509458Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:37:48.909078862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:38138"
ts=2026-05-05T11:37:55.987411912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T11:37:58.946170482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56994"
ts=2026-05-05T11:38:18.911250209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:33922"
ts=2026-05-05T11:38:25.98967807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:38:39.785992089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:38:55.991927007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T11:38:58.951358717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41644"
ts=2026-05-05T11:39:18.044861948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:56696"
ts=2026-05-05T11:39:25.99426561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:39:28.999779484Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:39:55.997335356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T11:40:25.999907668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:40:29.7846401Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:40:56.002277395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:41:15.061997039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:41:15.063901862Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:41:18.921186543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:53170"
ts=2026-05-05T11:41:26.004913494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T11:41:48.926435142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:43888"
ts=2026-05-05T11:41:56.007774246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:42:01.269269786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:42:26.0101369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T11:42:28.972902072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60056"
level=debug ts=2026-05-05T11:42:47.382661762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:42:56.012604688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T11:42:58.976406948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49438"
ts=2026-05-05T11:43:18.063795328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:43992"
ts=2026-05-05T11:43:18.93408266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56818"
ts=2026-05-05T11:43:26.014479243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:43:44.113660282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:43:48.936916329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46506"
ts=2026-05-05T11:43:56.016489958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T11:44:18.938835795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:58858"
ts=2026-05-05T11:44:26.019906998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:44:34.15439518Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:44:48.070367103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:40388"
ts=2026-05-05T11:44:56.022227237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T11:44:58.987955164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50148"
level=error ts=2026-05-05T11:45:21.486010678Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:45:21.488187717Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:45:21.490011525Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:45:21.490037569Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:45:26.024588202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T11:45:28.990372721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50422"
level=debug ts=2026-05-05T11:45:32.602952457Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:45:32.604737814Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:45:56.027047858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:46:17.874549987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:46:18.94831337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:33530"
ts=2026-05-05T11:46:26.028922899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T11:46:28.997155515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:37982"
ts=2026-05-05T11:46:48.082605764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:45028"
ts=2026-05-05T11:46:56.03136909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:47:00.094878674Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:47:18.085115297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:41472"
ts=2026-05-05T11:47:26.033449769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T11:47:48.087940959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:56666"
ts=2026-05-05T11:47:56.035831681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:47:58.057140561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:48:18.090658213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:35166"
ts=2026-05-05T11:48:26.038323994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T11:48:48.092612882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:33722"
ts=2026-05-05T11:48:56.042760837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:48:58.212071337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:48:59.013650962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42062"
ts=2026-05-05T11:49:18.963497098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46548"
ts=2026-05-05T11:49:26.046270312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T11:49:29.017054242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40586"
level=debug ts=2026-05-05T11:49:39.387304791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:49:39.389146885Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:49:56.048422213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:50:16.999520372Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:50:26.050418243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T11:50:48.971862783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:36666"
ts=2026-05-05T11:50:56.05315156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:50:58.730769398Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:51:18.101575995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:48558"
ts=2026-05-05T11:51:26.055618427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T11:51:29.029979002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50146"
level=debug ts=2026-05-05T11:51:33.051524938Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:51:48.977241336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:37846"
ts=2026-05-05T11:51:56.057797262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T11:52:18.105798007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:39400"
ts=2026-05-05T11:52:18.979750736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:58982"
level=debug ts=2026-05-05T11:52:24.038423861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:52:26.06037582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T11:52:56.063100197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T11:53:11.981332629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:53:11.982927127Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:53:18.110674708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:33716"
ts=2026-05-05T11:53:18.984401841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:34468"
ts=2026-05-05T11:53:26.065921748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T11:53:48.112590568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:38686"
ts=2026-05-05T11:53:48.986507165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:50670"
ts=2026-05-05T11:53:56.068024486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T11:53:59.045967794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48390"
level=debug ts=2026-05-05T11:54:13.09072454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:54:26.070270312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T11:54:29.049529063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:54390"
ts=2026-05-05T11:54:48.116097706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:59038"
level=debug ts=2026-05-05T11:54:49.379496824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:54:56.072669553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T11:54:59.052572788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45990"
ts=2026-05-05T11:55:18.994287323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:38154"
level=error ts=2026-05-05T11:55:21.485826753Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:55:21.487938388Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:55:21.48977496Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:55:21.489795112Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:55:26.074587485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:55:47.264413856Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:55:56.076293651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T11:55:59.058633913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60826"
level=debug ts=2026-05-05T11:56:25.200668396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:56:26.07882745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T11:56:56.080989307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T11:57:03.586516259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:57:03.588355787Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:57:19.006045425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:41388"
ts=2026-05-05T11:57:26.083584483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T11:57:29.067043502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35470"
ts=2026-05-05T11:57:56.085440725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:58:00.990832002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:58:26.088177259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:58:52.011788241Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:58:56.090676914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T11:59:19.015481819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:41652"
ts=2026-05-05T11:59:26.093314317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T11:59:49.017513537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:38766"
level=debug ts=2026-05-05T11:59:51.859715817Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:59:56.095494251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:00:26.09836071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:00:29.088067452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55010"
level=debug ts=2026-05-05T12:00:30.787084413Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:00:49.022852124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:53138"
ts=2026-05-05T12:00:56.10182332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:01:18.146079004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:55652"
ts=2026-05-05T12:01:19.025592513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:59298"
level=debug ts=2026-05-05T12:01:21.941718677Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:01:21.944008314Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:01:26.104762063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:01:56.107301116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T12:02:14.673509681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:02:19.031039137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:48148"
ts=2026-05-05T12:02:26.109799425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:02:29.115741994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42872"
ts=2026-05-05T12:02:48.150708634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:37428"
ts=2026-05-05T12:02:49.033142603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:47584"
ts=2026-05-05T12:02:56.112507306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:02:59.118923955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:36464"
level=debug ts=2026-05-05T12:03:06.90008605Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:03:26.11570954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:03:49.038595753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:54728"
level=debug ts=2026-05-05T12:03:51.214482122Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:03:56.117831618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:04:18.158870906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:34738"
ts=2026-05-05T12:04:26.120235814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:04:38.723511389Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:04:56.12280095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:04:59.133157656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58138"
level=error ts=2026-05-05T12:05:21.486472208Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:05:21.488290908Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:05:21.49002154Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:05:21.490040705Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:05:26.12569036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:05:26.541667784Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:05:26.54393964Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:05:29.136274787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:54078"
ts=2026-05-05T12:05:56.128220351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:05:59.138843933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49356"
level=debug ts=2026-05-05T12:06:09.218733373Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:06:19.051577791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:52952"
ts=2026-05-05T12:06:26.130448664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:06:29.142169066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:37340"
ts=2026-05-05T12:06:56.132117232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T12:06:57.126474603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:06:59.145509491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52974"
ts=2026-05-05T12:07:26.134459704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:07:29.148181351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45098"
level=debug ts=2026-05-05T12:07:34.610447424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:07:56.136668979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:08:16.839670618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:08:19.05939498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:52032"
ts=2026-05-05T12:08:26.138935525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:08:29.153664556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53138"
ts=2026-05-05T12:08:56.141645997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T12:09:13.565103112Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:09:13.567001528Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:09:26.144034949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:09:29.159263674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60896"
ts=2026-05-05T12:09:49.068091198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56110"
level=debug ts=2026-05-05T12:09:53.848820357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:09:56.146695318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:09:59.162557663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:54468"
ts=2026-05-05T12:10:26.148774535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:10:48.189242929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:45028"
level=debug ts=2026-05-05T12:10:50.685540361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:10:56.150890156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:11:18.191533855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:41308"
ts=2026-05-05T12:11:19.075040259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:34074"
ts=2026-05-05T12:11:26.153301573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:11:29.171545856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41616"
level=debug ts=2026-05-05T12:11:52.03226035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:11:56.156019795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:11:59.173869745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57858"
ts=2026-05-05T12:12:19.078239242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:52378"
ts=2026-05-05T12:12:26.157980868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:12:49.080783903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:48608"
level=debug ts=2026-05-05T12:12:51.061547815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:12:56.160610224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:13:18.199163768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:52888"
ts=2026-05-05T12:13:19.083050052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:59592"
ts=2026-05-05T12:13:26.163541285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:13:34.322244793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:13:34.323830727Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:13:56.166107756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:14:19.089256427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:43706"
ts=2026-05-05T12:14:26.16823427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T12:14:30.306064129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:14:56.171109269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:14:59.192166848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60360"
level=debug ts=2026-05-05T12:15:16.489309411Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:15:18.209985747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:54552"
level=error ts=2026-05-05T12:15:21.48330794Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:15:21.485692592Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:15:21.488411362Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:15:21.488435192Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:15:26.178380659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:15:56.180585703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:16:13.52638517Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:16:18.213411116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:54758"
ts=2026-05-05T12:16:19.103468663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:58330"
ts=2026-05-05T12:16:26.182284279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T12:16:51.220858663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:16:56.184551719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:17:26.186927071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:17:29.207879702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57310"
level=debug ts=2026-05-05T12:17:47.189462507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:17:47.191071477Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:17:56.189390086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T12:18:24.203469854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:18:26.191982197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:18:56.194594162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:18:59.217092165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38972"
level=debug ts=2026-05-05T12:18:59.375416086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:19:19.117828738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:35234"
ts=2026-05-05T12:19:26.196584019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:19:51.48348054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:19:56.198902421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:19:59.222291267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:37150"
ts=2026-05-05T12:20:26.200602016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T12:20:35.09584129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:20:56.203555499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:20:59.227993198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45472"
ts=2026-05-05T12:21:26.205913896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:21:29.230568931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33064"
level=debug ts=2026-05-05T12:21:37.025231336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:21:37.026779847Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:21:48.240882705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:57386"
ts=2026-05-05T12:21:56.207714366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:21:59.23355898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56078"
level=debug ts=2026-05-05T12:22:16.692797126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:22:26.209894408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:22:49.135959571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:54074"
ts=2026-05-05T12:22:56.212391601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:22:57.244142755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:22:59.23886341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50502"
ts=2026-05-05T12:23:19.138420518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:45710"
ts=2026-05-05T12:23:26.215070273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:23:48.250104191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:48128"
ts=2026-05-05T12:23:49.140474128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:44820"
level=debug ts=2026-05-05T12:23:55.365946742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:23:56.217339678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:23:59.24433961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:36918"
ts=2026-05-05T12:24:18.252233995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:51190"
ts=2026-05-05T12:24:26.220018666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:24:54.289470165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:24:56.222727161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:24:59.25063656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49272"
ts=2026-05-05T12:25:18.257875558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:51668"
level=error ts=2026-05-05T12:25:21.484802849Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:25:21.486779245Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:25:21.488704408Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:25:21.488725962Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:25:26.225135734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:25:41.645825721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:25:41.647661476Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:25:56.227134037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:26:19.152428367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:40550"
ts=2026-05-05T12:26:26.229426693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:26:29.260584483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39204"
level=debug ts=2026-05-05T12:26:40.793217323Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:26:56.23236761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:26:59.263174053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46372"
level=debug ts=2026-05-05T12:27:15.148513263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:27:19.156024206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:45246"
ts=2026-05-05T12:27:26.234407297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:27:48.269488973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:59394"
ts=2026-05-05T12:27:56.236327723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:27:59.268634398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53310"
level=debug ts=2026-05-05T12:28:01.152438184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:28:19.159019063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:54046"
ts=2026-05-05T12:28:26.238808374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:28:56.240933285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:28:59.274583235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59076"
level=debug ts=2026-05-05T12:29:02.478218354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:29:19.164409118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42936"
ts=2026-05-05T12:29:26.243462773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:29:40.820904917Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:29:40.822619341Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:29:56.24591347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:29:59.279737924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56558"
ts=2026-05-05T12:30:19.169696931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:35340"
ts=2026-05-05T12:30:26.248302937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:30:29.92518316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:30:48.28154055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:39954"
ts=2026-05-05T12:30:49.172166527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:48470"
ts=2026-05-05T12:30:56.250250534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:31:08.947425736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:31:26.25211188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:31:29.287431597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59910"
ts=2026-05-05T12:31:48.287316859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:44118"
ts=2026-05-05T12:31:56.254891372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:31:59.290078538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51140"
level=debug ts=2026-05-05T12:32:08.447114283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:32:19.176968809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:33902"
ts=2026-05-05T12:32:26.25719485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:32:52.556926264Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:32:56.26038147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:32:59.296416165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46590"
ts=2026-05-05T12:33:26.263127412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:33:29.299685588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60014"
level=debug ts=2026-05-05T12:33:42.243841987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:33:42.245832081Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:33:56.265029008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:33:59.302191066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56382"
ts=2026-05-05T12:34:26.266898369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:34:39.945174812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:34:48.300572949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:54080"
ts=2026-05-05T12:34:56.269717317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:34:59.307923205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59082"
level=error ts=2026-05-05T12:35:21.483273736Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:35:21.485501198Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:35:21.487434261Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:35:21.487470237Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T12:35:21.752252997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:35:26.272350617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:35:29.310838175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:43422"
ts=2026-05-05T12:35:48.304943756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:55094"
ts=2026-05-05T12:35:56.274838277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T12:36:18.019501431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:36:18.307012727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:46066"
ts=2026-05-05T12:36:26.277366089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:36:29.316128277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33644"
ts=2026-05-05T12:36:56.279978686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:36:57.794371873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:36:59.318693374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60758"
ts=2026-05-05T12:37:26.282134674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:37:29.321708216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57946"
ts=2026-05-05T12:37:48.315572269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:56786"
level=debug ts=2026-05-05T12:37:53.426279039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:37:53.428115072Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:37:56.284214747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:37:59.3239916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35988"
ts=2026-05-05T12:38:26.285889503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:38:29.32645764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51358"
level=debug ts=2026-05-05T12:38:44.517920716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:38:56.288624948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:39:26.291224338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:39:28.910660054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:39:48.326223567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:32984"
ts=2026-05-05T12:39:56.293427928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T12:40:10.321608489Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:40:18.328811597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:41776"
ts=2026-05-05T12:40:19.210837769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:45468"
ts=2026-05-05T12:40:26.295431366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:40:56.297492217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:41:02.364063706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:41:18.332080872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:53938"
ts=2026-05-05T12:41:26.300059395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:41:48.334172013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:49208"
ts=2026-05-05T12:41:56.302615961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T12:41:58.581340493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:41:58.583740987Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:42:19.223275759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:49874"
ts=2026-05-05T12:42:26.305265918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:42:50.130037098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:42:56.308327779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:43:26.310756215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:43:29.358515357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46164"
level=debug ts=2026-05-05T12:43:48.726020278Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:43:56.313180018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:44:19.232196269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:58372"
ts=2026-05-05T12:44:26.314933271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T12:44:46.245110301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:44:56.317841349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:44:59.366670789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41444"
ts=2026-05-05T12:45:18.3509527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:60126"
level=error ts=2026-05-05T12:45:21.485855443Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:45:21.487766789Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:45:21.489539439Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:45:21.489561525Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:45:26.320729855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:45:35.082703451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:45:48.353048855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:36176"
ts=2026-05-05T12:45:49.239171603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:41984"
ts=2026-05-05T12:45:56.32342917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:45:59.372715381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40092"
level=debug ts=2026-05-05T12:46:26.199564994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:46:26.201140668Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:46:26.326086502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:46:56.328670089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:46:59.37883586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:36610"
ts=2026-05-05T12:47:19.247578606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:54936"
level=debug ts=2026-05-05T12:47:19.584531245Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:47:26.331548525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:47:48.36134884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:60330"
ts=2026-05-05T12:47:56.33361532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T12:48:01.015664652Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:48:19.25309902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:39752"
ts=2026-05-05T12:48:26.335607841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:48:56.338502989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:48:59.390449001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39178"
level=debug ts=2026-05-05T12:49:00.353806124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:49:26.341254004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T12:49:38.282836251Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:49:48.371725069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:36856"
ts=2026-05-05T12:49:49.260403187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:51692"
ts=2026-05-05T12:49:56.343845431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:49:59.396331316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50066"
level=debug ts=2026-05-05T12:50:22.704795039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:50:22.706813396Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:50:26.346728374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:50:29.399882605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:43206"
ts=2026-05-05T12:50:48.37597736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:46864"
ts=2026-05-05T12:50:49.264872485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:55982"
ts=2026-05-05T12:50:56.348778455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T12:51:06.255259403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:51:18.378299336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:54270"
ts=2026-05-05T12:51:26.351084028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:51:48.380354253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:38528"
level=debug ts=2026-05-05T12:51:52.570949213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:51:56.353692526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:51:59.407834655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40634"
ts=2026-05-05T12:52:26.356086132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:52:29.410498443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42676"
level=debug ts=2026-05-05T12:52:54.991084988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:52:56.358829741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:52:59.413689367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60850"
ts=2026-05-05T12:53:18.38798072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:57490"
ts=2026-05-05T12:53:26.361222588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:53:56.363654145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:53:57.005944482Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:54:18.393631422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:40046"
ts=2026-05-05T12:54:26.365479521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T12:54:51.61882865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:54:51.62057607Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:54:56.368064518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:55:19.285047432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:53980"
level=error ts=2026-05-05T12:55:21.484140673Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:55:21.486156061Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:55:21.487932029Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:55:21.487958426Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:55:26.3703577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T12:55:38.694986854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:55:48.400529345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:45484"
ts=2026-05-05T12:55:56.372525039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
ts=2026-05-05T12:55:59.433094044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33914"
level=debug ts=2026-05-05T12:56:17.955754417Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:56:18.402247067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:56606"
ts=2026-05-05T12:56:26.37541374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:56:29.435557069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48224"
ts=2026-05-05T12:56:56.378295545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:56:59.437939493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60506"
level=debug ts=2026-05-05T12:57:10.252618639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:57:26.38065991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:57:56.382979426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:58:11.426174653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:58:18.411072108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:44350"
ts=2026-05-05T12:58:19.301610008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:40720"
ts=2026-05-05T12:58:26.385476682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T12:58:29.446703693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50536"
ts=2026-05-05T12:58:56.387798727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:59:11.379602539Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:59:11.380997342Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:59:26.390012264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T12:59:56.392038915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
level=debug ts=2026-05-05T13:00:10.224675314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T13:00:19.310151983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:47408"
ts=2026-05-05T13:00:26.394265526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T13:00:49.24133307Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T13:00:56.396350778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
ts=2026-05-05T13:00:59.461610906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48058"
ts=2026-05-05T13:01:26.398530248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2d81bcce 10.128.0.90:7946"
ts=2026-05-05T13:01:29.464160293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:43884"
level=debug ts=2026-05-05T13:01:35.17760158Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T13:01:48.424463035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:48540"
ts=2026-05-05T13:01:56.400967997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T13:02:22.135088845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T13:02:26.402990132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"