level=info ts=2026-05-06T13:18:32.615655099Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-06T13:18:32.615699507Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-06T13:18:32.616848011Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-06T13:18:32.617124418Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-06T13:18:32.617185351Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.72 inf=eth0
level=debug ts=2026-05-06T13:18:32.61789501Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-06T13:18:32.617965083Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.72 inf=eth0
level=info ts=2026-05-06T13:18:32.617990507Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-999165aa
ts=2026-05-06T13:18:32.618050123Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-06T13:18:32.618061282Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.72 advertisePort=7946
level=debug ts=2026-05-06T13:18:32.618075845Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.72 advertisePort=7946
level=info ts=2026-05-06T13:18:32.618183408Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-06T13:18:32.623115092Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-06T13:18:32.623131249Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-06T13:18:32.623145105Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=4.962869ms
level=error ts=2026-05-06T13:18:32.623156658Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-06T13:18:32.623170637Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-06T13:18:32.625426687Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T13:18:32.625441159Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-06T13:18:32.632031052Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-06T13:18:32.63205102Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-05-06T13:18:32.632071595Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-06T13:18:32.632058873Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-06T13:18:32.632081566Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-06T13:18:32.632096784Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-06T13:18:32.632105338Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-06T13:18:32.632189437Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-06T13:18:32.632205355Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-05-06T13:18:32.632328248Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-06T13:18:32.632406093Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-06T13:18:32.632396839Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-06T13:18:32.63245041Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-06T13:18:32.632610455Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-06T13:18:32.801951839Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-06T13:18:32.802019976Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-06T13:18:33.802822157Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-06T13:18:33.951338922Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-06T13:18:33.951395274Z caller=loki.go:615 msg="Loki started" startup_time=1.401246237s
ts=2026-05-06T13:18:34.031360132Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T13:18:34.031389157Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T13:18:37.227814575Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T13:18:37.227837993Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-06T13:18:38.952324835Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-06T13:18:38.952379126Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-06T13:18:43.105607691Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T13:18:43.105634168Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T13:18:52.347936071Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.78:7946: dial tcp 10.128.0.78:7946: connect: connection refused"
ts=2026-05-06T13:18:52.34899734Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.74:7946: dial tcp 10.129.0.74:7946: connect: connection refused"
ts=2026-05-06T13:18:52.349719649Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.71:7946: dial tcp 10.129.0.71:7946: connect: connection refused"
ts=2026-05-06T13:18:52.349816192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.72:7946"
ts=2026-05-06T13:18:52.349838596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:52030"
ts=2026-05-06T13:18:52.351265809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.83:7946"
ts=2026-05-06T13:18:52.352970767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.86:7946"
level=info ts=2026-05-06T13:18:52.353549911Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=19.73037246s
level=debug ts=2026-05-06T13:18:54.257216597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T13:18:55.743850096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:38576"
ts=2026-05-06T13:18:58.876585097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:41330"
ts=2026-05-06T13:19:01.608348068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:46286"
level=debug ts=2026-05-06T13:19:08.079658874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-06T13:19:25.906394853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T13:19:28.788483944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T13:19:44.864187282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T13:19:55.875869075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:39972"
ts=2026-05-06T13:19:58.790312564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:20:03.361398345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:47388"
ts=2026-05-06T13:20:04.676751274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:50146"
level=debug ts=2026-05-06T13:20:18.769210029Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-06T13:20:18.770778609Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T13:20:28.791947515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T13:20:55.879539811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:38020"
ts=2026-05-06T13:20:58.793577645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:21:03.365584549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:55538"
level=debug ts=2026-05-06T13:21:04.233224188Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T13:21:28.794833484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T13:21:58.796278094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T13:22:07.763101104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T13:22:28.797656426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T13:22:55.8878678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:38942"
ts=2026-05-06T13:22:58.799255757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T13:23:03.932938331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T13:23:25.889393583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:47470"
ts=2026-05-06T13:23:28.800571406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T13:23:58.802087737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T13:24:09.689893795Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T13:24:25.893353351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:50460"
ts=2026-05-06T13:24:28.803753651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:24:34.694217639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:59972"
ts=2026-05-06T13:24:55.895120047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:36396"
level=debug ts=2026-05-06T13:24:58.538320532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-06T13:24:58.540101338Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T13:24:58.805647932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:25:04.69545698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34852"
ts=2026-05-06T13:25:28.807563086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:25:33.379064581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:54910"
ts=2026-05-06T13:25:58.809070985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T13:25:59.838580365Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:26:25.899332201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:46100"
ts=2026-05-06T13:26:28.81058979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:26:34.700732012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:45988"
ts=2026-05-06T13:26:55.90171855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:37790"
level=debug ts=2026-05-06T13:26:56.655190202Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:26:58.812845969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:27:28.814185633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
level=debug ts=2026-05-06T13:27:48.340269873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:27:58.815658007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T13:28:04.706754804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56102"
ts=2026-05-06T13:28:28.817676978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:28:34.708273504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:42490"
level=info ts=2026-05-06T13:28:38.953351714Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-06T13:28:38.957699305Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T13:28:38.959895039Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T13:28:38.961789068Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T13:28:38.961815927Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-06T13:28:38.961836127Z caller=tables_manager.go:148 msg="compactor started"
level=debug ts=2026-05-06T13:28:45.065327392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:28:58.819770555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T13:29:03.393698681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:54564"
ts=2026-05-06T13:29:04.709850216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34342"
level=debug ts=2026-05-06T13:29:20.80400906Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T13:29:20.805767327Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T13:29:25.910297479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:38972"
ts=2026-05-06T13:29:28.821622215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T13:29:34.710991426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:51462"
ts=2026-05-06T13:29:55.912282987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:56556"
ts=2026-05-06T13:29:58.823299604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T13:29:59.500945689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:30:04.712691296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:53970"
ts=2026-05-06T13:30:25.914462831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:55770"
ts=2026-05-06T13:30:28.826504219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:30:34.714237226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:38436"
level=debug ts=2026-05-06T13:30:58.010701571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:30:58.828113024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T13:31:03.401949179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:51952"
ts=2026-05-06T13:31:28.830198893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T13:31:58.831207505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
level=debug ts=2026-05-06T13:31:58.990742287Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:32:28.832476221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T13:32:33.407584956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:52234"
level=debug ts=2026-05-06T13:32:40.0394522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:32:55.923461392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:57782"
ts=2026-05-06T13:32:58.835352386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T13:33:03.409882307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:50418"
ts=2026-05-06T13:33:28.837739785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T13:33:33.411953255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:59270"
level=debug ts=2026-05-06T13:33:38.483509038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T13:33:38.48568003Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T13:33:58.839605258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:34:03.413812588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:52362"
ts=2026-05-06T13:34:28.841268011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
level=debug ts=2026-05-06T13:34:36.791028464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:34:58.84247564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T13:35:04.74327191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:60170"
ts=2026-05-06T13:35:28.843791308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T13:35:33.418694141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:40444"
level=debug ts=2026-05-06T13:35:34.59867817Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:35:55.93281224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:43480"
ts=2026-05-06T13:35:58.845763028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:36:28.847623094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T13:36:32.594251287Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:36:33.421596292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:57358"
ts=2026-05-06T13:36:55.936041587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:35262"
ts=2026-05-06T13:36:58.849549126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:37:25.938107995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:45226"
ts=2026-05-06T13:37:28.85137966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T13:37:31.528122751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:37:33.426507146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:37090"
ts=2026-05-06T13:37:58.853216195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T13:38:04.755667078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:43072"
level=debug ts=2026-05-06T13:38:26.260055675Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T13:38:26.261646162Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T13:38:28.854304775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
level=error ts=2026-05-06T13:38:38.966440336Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T13:38:38.968316611Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T13:38:38.970071475Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T13:38:38.970095655Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T13:38:55.943687876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:34210"
ts=2026-05-06T13:38:58.856145655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
level=debug ts=2026-05-06T13:39:12.711838108Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:39:28.8572028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T13:39:33.433087722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:49580"
ts=2026-05-06T13:39:34.764407514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58158"
level=debug ts=2026-05-06T13:39:57.332074893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:39:58.859127707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T13:40:04.765485981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56494"
ts=2026-05-06T13:40:25.949610729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:48964"
ts=2026-05-06T13:40:28.862824086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T13:40:43.909951729Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:40:55.951700372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:33042"
ts=2026-05-06T13:40:58.866091875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T13:41:25.904561782Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:41:25.953844246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:38294"
ts=2026-05-06T13:41:28.868206773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T13:41:33.439727224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:54884"
ts=2026-05-06T13:41:58.870283398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T13:42:14.103112176Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T13:42:14.105464383Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T13:42:28.872687449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T13:42:33.443893629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:58708"
ts=2026-05-06T13:42:58.874554351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T13:43:04.777830824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:51310"
level=debug ts=2026-05-06T13:43:14.016170737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:43:28.876240326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T13:43:58.878152004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T13:44:03.236676629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:44:28.879943166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T13:44:33.451655465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:33814"
ts=2026-05-06T13:44:34.784067993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58428"
level=debug ts=2026-05-06T13:44:42.754855548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:44:58.881594242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T13:45:18.59221874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:45:28.883256801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:45:58.885028098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T13:46:15.9960255Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T13:46:15.997795556Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T13:46:28.886663229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:46:34.791343525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:54574"
ts=2026-05-06T13:46:55.975945202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:33880"
ts=2026-05-06T13:46:58.888608743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T13:47:03.46078871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:34956"
level=debug ts=2026-05-06T13:47:06.730992505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:47:28.890755524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:47:33.462940538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:36228"
ts=2026-05-06T13:47:58.892570196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T13:48:03.464531913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:50530"
ts=2026-05-06T13:48:04.797559222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:42834"
level=debug ts=2026-05-06T13:48:07.886324995Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:48:28.893682932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:48:33.466810759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:54150"
level=error ts=2026-05-06T13:48:38.966454042Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T13:48:38.968506738Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T13:48:38.970303728Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T13:48:38.97032916Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T13:48:58.895302525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T13:49:03.249836247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:49:03.468390895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:38806"
ts=2026-05-06T13:49:04.801814565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34542"
ts=2026-05-06T13:49:28.896903343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T13:49:33.470799223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:56372"
ts=2026-05-06T13:49:34.809502849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34448"
level=debug ts=2026-05-06T13:49:45.887213923Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:49:58.898357565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T13:50:21.534133802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T13:50:21.536174806Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T13:50:25.986364192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:35634"
ts=2026-05-06T13:50:28.899893348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T13:50:55.988725635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:51856"
ts=2026-05-06T13:50:58.901691324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T13:51:09.078488099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:51:28.904100311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:51:58.909599224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T13:52:03.479763185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:45874"
ts=2026-05-06T13:52:04.819944011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58776"
level=debug ts=2026-05-06T13:52:10.762019942Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:52:25.993806371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:57938"
ts=2026-05-06T13:52:28.911965708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T13:52:58.914478995Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T13:53:01.090101098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:53:03.484523704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:43838"
ts=2026-05-06T13:53:28.916342166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T13:53:53.211028505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:53:55.999642622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:57370"
ts=2026-05-06T13:53:58.917728502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:54:03.487943088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:50738"
ts=2026-05-06T13:54:28.91957514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:54:33.489349792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:59298"
level=debug ts=2026-05-06T13:54:47.515129731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T13:54:47.517040964Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T13:54:56.00375657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:51390"
ts=2026-05-06T13:54:58.921830708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:55:03.49091808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:51364"
ts=2026-05-06T13:55:28.924356706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T13:55:44.299260531Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:55:58.926951807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T13:56:26.01122336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:37506"
ts=2026-05-06T13:56:28.928950734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T13:56:33.496314922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:48918"
level=debug ts=2026-05-06T13:56:33.701433652Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:56:34.837051465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:35630"
ts=2026-05-06T13:56:58.930385964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T13:57:03.49835419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:56226"
level=debug ts=2026-05-06T13:57:28.329054593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:57:28.932129846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T13:57:33.499899117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:37994"
ts=2026-05-06T13:57:58.933762367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T13:58:04.842124954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:46810"
level=debug ts=2026-05-06T13:58:09.092559959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:58:28.936043344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=error ts=2026-05-06T13:58:38.966208688Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T13:58:38.968087422Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T13:58:38.969856666Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T13:58:38.969875295Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T13:58:58.937693793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T13:59:08.248010607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T13:59:08.249698006Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T13:59:26.021797777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:40188"
ts=2026-05-06T13:59:28.939334234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
level=debug ts=2026-05-06T13:59:55.621434942Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T13:59:58.940532943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:00:04.849500465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:57832"
ts=2026-05-06T14:00:28.942684001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:00:33.510840341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:55778"
level=debug ts=2026-05-06T14:00:46.640118689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:00:58.943979264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T14:01:28.946132848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T14:01:36.239904118Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:01:58.947727916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T14:02:16.071249126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:02:26.031641019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:36564"
ts=2026-05-06T14:02:28.949492251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:02:58.950845515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:03:03.518366818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:59046"
ts=2026-05-06T14:03:04.860302563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:33538"
level=debug ts=2026-05-06T14:03:16.242381241Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T14:03:16.244611795Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T14:03:28.95204248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:03:33.520350004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:42662"
ts=2026-05-06T14:03:34.861431349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34722"
ts=2026-05-06T14:03:56.037573895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:33698"
ts=2026-05-06T14:03:58.953545214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T14:04:11.059136019Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:04:28.955330075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T14:04:34.865547207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:59898"
level=debug ts=2026-05-06T14:04:55.927738626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:04:56.041337844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:46354"
ts=2026-05-06T14:04:58.957604636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:05:04.866904464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:51602"
ts=2026-05-06T14:05:28.959182966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T14:05:32.592378772Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:05:33.527670575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:47530"
ts=2026-05-06T14:05:58.960855248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T14:06:21.361769602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:06:28.962677939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:06:33.530238857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:34584"
ts=2026-05-06T14:06:34.873085119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34096"
ts=2026-05-06T14:06:56.048737229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:35278"
level=debug ts=2026-05-06T14:06:57.187603277Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T14:06:57.189219458Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T14:06:58.964346761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T14:07:03.531917476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:43668"
ts=2026-05-06T14:07:26.050607792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:44140"
ts=2026-05-06T14:07:28.966081083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T14:07:37.741043854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:07:58.96772249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T14:08:03.535456626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:56890"
level=debug ts=2026-05-06T14:08:27.296733882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:08:28.969218481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:08:34.880529377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34600"
level=error ts=2026-05-06T14:08:38.965617308Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T14:08:38.967893467Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T14:08:38.970291784Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T14:08:38.970317131Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T14:08:56.055642151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:56128"
ts=2026-05-06T14:08:58.97074074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T14:09:14.218257465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:09:28.972328023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T14:09:58.155810214Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:09:58.974443591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:10:04.885881378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:42694"
ts=2026-05-06T14:10:28.976330259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T14:10:33.54447803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:41138"
level=debug ts=2026-05-06T14:10:44.305126504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T14:10:44.306945917Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T14:10:56.063752686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:52228"
ts=2026-05-06T14:10:58.977836679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T14:11:19.037499785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:11:26.065328232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:51934"
ts=2026-05-06T14:11:28.978999081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:11:58.980838246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T14:12:16.150253594Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:12:28.982810557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:12:58.984033021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:13:03.551306038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:57618"
ts=2026-05-06T14:13:04.897987609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:47050"
level=debug ts=2026-05-06T14:13:07.844226453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:13:28.985919488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T14:13:33.55302819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:45250"
ts=2026-05-06T14:13:56.073765776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:33212"
ts=2026-05-06T14:13:58.988229795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T14:14:08.861704283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:14:26.076088101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:48758"
ts=2026-05-06T14:14:28.989629891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
level=debug ts=2026-05-06T14:14:54.807062839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T14:14:54.808614654Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T14:14:58.990809078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:15:03.557797576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:35738"
ts=2026-05-06T14:15:26.079244994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:40984"
ts=2026-05-06T14:15:28.993169847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T14:15:33.559711008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:43478"
level=debug ts=2026-05-06T14:15:51.048764533Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:15:58.995134693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T14:16:04.909596765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:49582"
ts=2026-05-06T14:16:26.082364484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:48342"
level=debug ts=2026-05-06T14:16:28.152015854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:16:28.996753387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:16:33.562697977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:43012"
ts=2026-05-06T14:16:58.998666335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:17:04.912715306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:54996"
level=debug ts=2026-05-06T14:17:24.722964977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:17:26.085883096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:34704"
ts=2026-05-06T14:17:29.000182836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T14:17:33.56537584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:50010"
ts=2026-05-06T14:17:59.00249913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T14:18:25.279036197Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:18:26.089504625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:37138"
ts=2026-05-06T14:18:29.003842461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:18:33.56871985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:59394"
level=error ts=2026-05-06T14:18:38.965719162Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T14:18:38.968489318Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T14:18:38.970359766Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T14:18:38.970386427Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T14:18:56.09154789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:56900"
ts=2026-05-06T14:18:59.005066559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T14:19:09.98826062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T14:19:09.989901738Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T14:19:29.006651905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T14:19:48.043817091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:19:59.008528103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:20:26.097262436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:47826"
ts=2026-05-06T14:20:29.01038777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T14:20:31.316437344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:20:34.926991646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:59752"
ts=2026-05-06T14:20:59.01224255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:21:03.576721773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:41200"
ts=2026-05-06T14:21:26.100364375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:46658"
ts=2026-05-06T14:21:29.014206603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
level=debug ts=2026-05-06T14:21:33.523923959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:21:34.930284339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:37316"
ts=2026-05-06T14:21:59.016112896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:22:03.579396118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:38272"
level=debug ts=2026-05-06T14:22:22.452878647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:22:29.017165334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:22:33.580908757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:46270"
ts=2026-05-06T14:22:56.106313075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:39644"
ts=2026-05-06T14:22:59.018877114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T14:23:05.447992778Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T14:23:05.449476005Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T14:23:26.108538311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:56808"
ts=2026-05-06T14:23:29.020254024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:23:33.583316039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:35926"
level=debug ts=2026-05-06T14:23:40.226053751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:23:59.021377158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:24:26.112881437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:59820"
ts=2026-05-06T14:24:29.022950016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T14:24:31.043331225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:24:33.587729774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:43906"
ts=2026-05-06T14:24:34.941807071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:57032"
ts=2026-05-06T14:24:59.025382888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:25:03.589994723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:54624"
ts=2026-05-06T14:25:26.116232746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:49238"
level=debug ts=2026-05-06T14:25:28.760141937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:25:29.027640075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:25:59.029368168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:26:03.593147491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:51316"
ts=2026-05-06T14:26:04.947208073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:45558"
ts=2026-05-06T14:26:29.030881258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T14:26:29.524585186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:26:59.032354476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T14:27:22.004297985Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T14:27:22.006031263Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T14:27:29.033963105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:27:56.124946022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:41802"
ts=2026-05-06T14:27:59.035969214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:28:03.600245894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:56558"
level=debug ts=2026-05-06T14:28:19.199277327Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:28:29.037936006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=error ts=2026-05-06T14:28:38.966529506Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T14:28:38.969328311Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T14:28:38.971206704Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T14:28:38.971231599Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T14:28:56.128926738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:53966"
ts=2026-05-06T14:28:59.040214776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T14:29:03.828140184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:29:29.041742593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:29:33.603907322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:59622"
ts=2026-05-06T14:29:34.960188169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:51744"
level=debug ts=2026-05-06T14:29:44.607702952Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:29:59.043248757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T14:30:21.992179597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:30:29.045369575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:30:33.607050218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:43870"
ts=2026-05-06T14:30:59.047027996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T14:31:03.608802575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:47006"
ts=2026-05-06T14:31:04.965831389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34814"
level=debug ts=2026-05-06T14:31:09.500960335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T14:31:09.50275604Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T14:31:26.138473542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:47226"
ts=2026-05-06T14:31:29.048347935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
level=debug ts=2026-05-06T14:31:47.251473896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:31:56.140114131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:55344"
ts=2026-05-06T14:31:59.049650808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:32:29.051068111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:32:33.613720946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:41406"
level=debug ts=2026-05-06T14:32:47.669939677Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:32:59.052681374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
level=debug ts=2026-05-06T14:33:24.569683818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:33:29.053644624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:33:59.055300514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T14:34:03.460951101Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:34:04.976809863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:45594"
ts=2026-05-06T14:34:29.057524677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T14:34:41.411221724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T14:34:41.413068541Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T14:34:59.059278979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:35:29.061654012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T14:35:42.148924554Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:35:56.154940534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:38990"
ts=2026-05-06T14:35:59.063507622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:36:04.984918849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:37410"
ts=2026-05-06T14:36:29.06582573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:36:34.986352056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:39386"
level=debug ts=2026-05-06T14:36:41.378159284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:36:56.158778119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:53056"
ts=2026-05-06T14:36:59.067676333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:37:04.988134318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:57328"
level=debug ts=2026-05-06T14:37:19.891208847Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:37:29.069996963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:37:59.072439129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:38:03.630049998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:48648"
ts=2026-05-06T14:38:04.991112367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:36922"
level=debug ts=2026-05-06T14:38:19.204476512Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:38:26.164251426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:53146"
ts=2026-05-06T14:38:29.07446189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=error ts=2026-05-06T14:38:38.965623034Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T14:38:38.968496746Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T14:38:38.970319055Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T14:38:38.970347218Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-06T14:38:56.741329302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T14:38:56.743244206Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T14:38:59.076205122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T14:39:03.633583423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:39232"
ts=2026-05-06T14:39:04.995292202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:55642"
ts=2026-05-06T14:39:26.167605505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:34130"
ts=2026-05-06T14:39:29.078000037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T14:39:58.662436338Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:39:59.079582366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T14:40:03.638160371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:54626"
ts=2026-05-06T14:40:29.081056333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:40:33.640372486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:53092"
level=debug ts=2026-05-06T14:40:41.444533375Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:40:59.082588224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T14:41:05.00259577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:33108"
ts=2026-05-06T14:41:29.08464937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T14:41:32.438703017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:41:59.086222304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:42:05.005507783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:47644"
level=debug ts=2026-05-06T14:42:15.754042612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:42:29.088363371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:42:35.007148377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:37674"
ts=2026-05-06T14:42:59.090461319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:43:03.648234637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:33708"
level=debug ts=2026-05-06T14:43:05.710125596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T14:43:05.71162785Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T14:43:29.092314577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T14:43:50.69770453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:43:56.180574195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:58028"
ts=2026-05-06T14:43:59.093971278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:44:03.652323418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:38570"
ts=2026-05-06T14:44:29.095962445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T14:44:33.254977619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:44:35.014484917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:53744"
ts=2026-05-06T14:44:59.098070047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T14:45:03.656221646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:34506"
level=debug ts=2026-05-06T14:45:18.358632071Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:45:29.099652927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:45:59.101030606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:46:03.660149411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:37512"
level=debug ts=2026-05-06T14:46:14.406605068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:46:29.102276042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:46:59.103766182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
level=debug ts=2026-05-06T14:46:59.448568896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T14:46:59.450187139Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T14:47:29.105354514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T14:47:40.537564805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:47:59.106602255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:48:26.1950844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:38030"
ts=2026-05-06T14:48:29.108004991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:48:35.028013028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:51768"
level=debug ts=2026-05-06T14:48:35.961207748Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T14:48:38.964864763Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T14:48:38.966654145Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T14:48:38.969668381Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T14:48:38.969695747Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T14:48:59.109444281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
level=debug ts=2026-05-06T14:49:20.230633443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:49:29.111094144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:49:56.200599874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:42260"
ts=2026-05-06T14:49:59.112515674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T14:50:00.250199055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:50:03.674714155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:53808"
ts=2026-05-06T14:50:26.202698818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:36198"
ts=2026-05-06T14:50:29.114496813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
ts=2026-05-06T14:50:35.035468023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:48162"
level=debug ts=2026-05-06T14:50:52.817191617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T14:50:52.818803295Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T14:50:56.205104341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:43880"
ts=2026-05-06T14:50:59.117849704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
ts=2026-05-06T14:51:26.207385262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:38280"
ts=2026-05-06T14:51:29.119330406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80176d8f 10.129.0.73:7946"
level=debug ts=2026-05-06T14:51:38.02254276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:51:59.1213538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
level=debug ts=2026-05-06T14:52:25.678089719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T14:52:26.211671454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:54728"
ts=2026-05-06T14:52:29.123192191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-46tcb-6b822fc6 10.130.0.86:7946"
ts=2026-05-06T14:52:59.124714789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-k62vn-1cf35604 10.130.0.83:7946"
level=debug ts=2026-05-06T14:53:14.754061774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"