level=info ts=2026-05-05T11:25:18.361570291Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-05T11:25:18.361606892Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-05T11:25:18.364171634Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-05T11:25:18.364604805Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-05T11:25:18.364697772Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.63 inf=eth0
level=debug ts=2026-05-05T11:25:18.365298792Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-05T11:25:18.365375179Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.63 inf=eth0
level=info ts=2026-05-05T11:25:18.365426065Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-3fba1526
ts=2026-05-05T11:25:18.365528362Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-05T11:25:18.365540023Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.63 advertisePort=7946
level=debug ts=2026-05-05T11:25:18.365556696Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.63 advertisePort=7946
level=info ts=2026-05-05T11:25:18.365666333Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-05T11:25:18.372079585Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T11:25:18.372097339Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-05T11:25:18.372108327Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.444269ms
level=error ts=2026-05-05T11:25:18.372117283Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-05T11:25:18.372128465Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-05T11:25:18.374127817Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T11:25:18.374141197Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-05T11:25:18.384068586Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-05T11:25:18.384077703Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-05T11:25:18.38410115Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-05T11:25:18.384114828Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-05T11:25:18.384129761Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-05T11:25:18.384153155Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-05T11:25:18.384183897Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-05T11:25:18.384191894Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-05T11:25:18.384383342Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-05T11:25:18.384390868Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-05T11:25:18.384440184Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-05T11:25:18.384455433Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-05T11:25:18.384484132Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-05T11:25:18.38459975Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-05T11:25:18.490295713Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-05T11:25:18.490363738Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-05T11:25:19.491109292Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-05T11:25:19.630493677Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-05T11:25:19.630552467Z caller=loki.go:615 msg="Loki started" startup_time=1.336546422s
ts=2026-05-05T11:25:20.058175614Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T11:25:20.058200006Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:25:22.477083364Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T11:25:22.477105292Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-05T11:25:24.630764823Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-05T11:25:24.630847166Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-05T11:25:29.072785557Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T11:25:29.072810497Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-05T11:25:35.250987032Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:25:38.187189093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.83:7946"
ts=2026-05-05T11:25:38.190741901Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.84:7946: dial tcp 10.128.0.84:7946: connect: connection refused"
ts=2026-05-05T11:25:38.192147547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.86:7946"
ts=2026-05-05T11:25:38.194910743Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.85:7946: dial tcp 10.128.0.85:7946: connect: connection refused"
ts=2026-05-05T11:25:38.195044667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.63:7946"
ts=2026-05-05T11:25:38.195089947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:41914"
level=info ts=2026-05-05T11:25:38.195304619Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=19.823171732s
ts=2026-05-05T11:25:42.991810589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:50556"
ts=2026-05-05T11:25:43.787836046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40150"
ts=2026-05-05T11:25:44.695862367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:48338"
level=debug ts=2026-05-05T11:25:53.381108437Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:25:59.116926148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:26:10.050665322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36262"
level=debug ts=2026-05-05T11:26:11.79256024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:26:29.119859164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T11:26:35.200938874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:26:40.053630029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:48882"
ts=2026-05-05T11:26:59.123235842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T11:27:02.942142588Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-05T11:27:02.943877858Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:27:29.126229229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T11:27:51.635976593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:27:59.129291304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T11:28:03.802420563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41734"
ts=2026-05-05T11:28:29.132082217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:28:33.805346193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43254"
ts=2026-05-05T11:28:59.134859081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
level=debug ts=2026-05-05T11:29:02.733663693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:29:03.80872863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44856"
ts=2026-05-05T11:29:29.137343099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T11:29:29.194946359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46948"
ts=2026-05-05T11:29:59.140084093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T11:30:03.81361932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57692"
level=debug ts=2026-05-05T11:30:08.063645466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:30:29.143518156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:30:33.816411444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54684"
ts=2026-05-05T11:30:40.070638772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:40598"
ts=2026-05-05T11:30:59.147051322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:30:59.202764424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34154"
ts=2026-05-05T11:31:10.073524349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:58092"
level=debug ts=2026-05-05T11:31:15.943725277Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:31:29.150403845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T11:31:59.153096363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:31:59.207499246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:55192"
ts=2026-05-05T11:32:03.82253491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48188"
level=debug ts=2026-05-05T11:32:23.5419531Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-05T11:32:23.543854507Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:32:29.156009572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T11:32:33.825056704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:60760"
ts=2026-05-05T11:32:40.079572444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38314"
ts=2026-05-05T11:32:59.158948892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T11:33:00.539536342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:33:03.827723604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35734"
ts=2026-05-05T11:33:10.082504336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:41796"
ts=2026-05-05T11:33:29.161904369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T11:33:29.214243986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47304"
ts=2026-05-05T11:33:33.83062559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:47852"
level=debug ts=2026-05-05T11:33:49.7199348Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:33:59.165493634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:34:29.168606495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T11:34:48.834675949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:34:59.171703701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T11:34:59.221809184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52240"
ts=2026-05-05T11:35:03.839687924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41928"
ts=2026-05-05T11:35:10.089643264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:49128"
level=info ts=2026-05-05T11:35:24.631869365Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-05T11:35:24.638913289Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:35:24.640924173Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:35:24.642807831Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:35:24.64283344Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T11:35:24.642847586Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-05T11:35:29.174414982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
level=debug ts=2026-05-05T11:35:42.549893457Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:35:59.177491497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T11:35:59.2261164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:37512"
ts=2026-05-05T11:36:03.844961208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:47778"
level=debug ts=2026-05-05T11:36:26.43375485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:36:26.435212933Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:36:29.180383464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T11:36:33.847726295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35250"
ts=2026-05-05T11:36:59.183011457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T11:37:03.851115255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:32812"
level=debug ts=2026-05-05T11:37:27.491464841Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:37:29.186128582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:37:33.854148172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41878"
ts=2026-05-05T11:37:59.188925336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
level=debug ts=2026-05-05T11:38:22.336802471Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:38:29.191875137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T11:38:59.194909275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T11:38:59.240080019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:44536"
level=debug ts=2026-05-05T11:39:04.306045321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:39:29.197563257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T11:39:29.243032636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:48776"
ts=2026-05-05T11:39:33.862550264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42958"
ts=2026-05-05T11:39:40.105616471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:41872"
level=debug ts=2026-05-05T11:39:55.964582873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:39:59.200859089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T11:40:03.865109704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48064"
ts=2026-05-05T11:40:29.203942343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T11:40:33.867809654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48020"
level=debug ts=2026-05-05T11:40:42.424121698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:40:42.426375594Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:40:59.206838436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T11:41:03.870327625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:45726"
level=debug ts=2026-05-05T11:41:20.831992565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:41:29.209867939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T11:41:33.872807577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50192"
ts=2026-05-05T11:41:59.213110085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:41:59.254644073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39810"
level=debug ts=2026-05-05T11:42:00.080817692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:42:29.216819163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T11:42:29.257803581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:37656"
level=debug ts=2026-05-05T11:42:40.137526841Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:42:59.219829259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:43:29.222909919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T11:43:35.574488815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:43:40.120594688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:46400"
ts=2026-05-05T11:43:59.226685959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T11:44:24.634204452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:44:24.636790205Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:44:29.230629932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:44:59.233709144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:44:59.270310429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39052"
ts=2026-05-05T11:45:03.887640508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43262"
level=debug ts=2026-05-05T11:45:12.398695141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:45:24.646382706Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:45:24.648370784Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:45:24.65000313Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:45:24.650022825Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:45:29.236446482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T11:45:55.406113169Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:45:59.239823272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:46:10.132879109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:56972"
ts=2026-05-05T11:46:29.243271831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T11:46:29.27811904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49500"
ts=2026-05-05T11:46:40.136226752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38078"
level=debug ts=2026-05-05T11:46:55.772951433Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:46:59.246797745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:46:59.281709885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33306"
ts=2026-05-05T11:47:03.895001978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:38762"
ts=2026-05-05T11:47:10.139371673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:41586"
ts=2026-05-05T11:47:29.249858878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
level=debug ts=2026-05-05T11:47:44.43478438Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:47:59.252546372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:47:59.287224768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53780"
ts=2026-05-05T11:48:10.143894232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37216"
ts=2026-05-05T11:48:29.258520945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T11:48:34.414094411Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:48:34.416280186Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:48:40.14634092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:43338"
ts=2026-05-05T11:48:59.261223928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T11:49:10.149331315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:45242"
level=debug ts=2026-05-05T11:49:14.904614121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:49:29.263604504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T11:49:40.152698171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36836"
ts=2026-05-05T11:49:59.266870638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T11:50:12.152051379Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:50:29.269915111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:50:33.908340043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50480"
ts=2026-05-05T11:50:59.272963925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:50:59.300840481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53538"
level=debug ts=2026-05-05T11:50:59.629026262Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:51:10.158849045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36518"
ts=2026-05-05T11:51:29.276266277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T11:51:29.303758733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:57764"
ts=2026-05-05T11:51:33.913914523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54296"
level=debug ts=2026-05-05T11:51:47.771487404Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:51:59.278978892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:52:03.916324383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:47406"
level=debug ts=2026-05-05T11:52:27.818555608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:52:27.820849113Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:52:29.282058668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T11:52:33.919335573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43744"
ts=2026-05-05T11:52:59.285164794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:53:03.92257709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54294"
level=debug ts=2026-05-05T11:53:04.443008664Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:53:29.288147768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T11:53:33.926012307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58684"
level=debug ts=2026-05-05T11:53:49.457601094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:53:59.290513381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T11:54:03.928600641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35804"
ts=2026-05-05T11:54:29.293929764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T11:54:33.632018648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:54:40.169882676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:54654"
ts=2026-05-05T11:54:59.297281823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:54:59.319934568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41154"
level=debug ts=2026-05-05T11:55:17.643071602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:55:24.645917243Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:55:24.647597521Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:55:24.649347955Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T11:55:24.649369948Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:55:29.300242439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:55:29.322659853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:56580"
ts=2026-05-05T11:55:59.303600413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T11:55:59.325746363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:50390"
level=debug ts=2026-05-05T11:56:15.336058593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:56:15.337672533Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T11:56:29.30586822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T11:56:33.938560727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:36906"
ts=2026-05-05T11:56:59.310635769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
level=debug ts=2026-05-05T11:57:11.631725736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:57:29.313608334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:57:29.335859922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34914"
ts=2026-05-05T11:57:40.182007435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:56718"
ts=2026-05-05T11:57:59.316917437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T11:58:11.21252214Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:58:29.320147471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T11:58:29.340933869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:56310"
ts=2026-05-05T11:58:40.187916928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36440"
ts=2026-05-05T11:58:59.323437169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T11:59:12.849965472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:59:29.326880613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T11:59:33.949955304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:38114"
ts=2026-05-05T11:59:40.19383021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37904"
level=debug ts=2026-05-05T11:59:48.920458212Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:59:59.329553618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:00:03.952921324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:32776"
ts=2026-05-05T12:00:29.33250794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:00:29.350464135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:44476"
ts=2026-05-05T12:00:40.198642275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:60838"
level=debug ts=2026-05-05T12:00:46.926578218Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:00:46.928861828Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:00:59.335716045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:00:59.353896525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33942"
ts=2026-05-05T12:01:03.958189533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56038"
level=debug ts=2026-05-05T12:01:21.236374596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:01:29.338828991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:01:29.357267421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39648"
ts=2026-05-05T12:01:59.342159996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:01:59.361024152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:48764"
level=debug ts=2026-05-05T12:02:00.100492633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:02:03.96340985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58252"
ts=2026-05-05T12:02:10.205144593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:60662"
ts=2026-05-05T12:02:29.345723345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:02:40.208082672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36084"
level=debug ts=2026-05-05T12:02:48.07298394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:02:59.348797374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:02:59.366635877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:56272"
ts=2026-05-05T12:03:29.352122163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:03:40.213293317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:59568"
level=debug ts=2026-05-05T12:03:43.344440519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:03:59.355259925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:04:03.97148142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42972"
level=debug ts=2026-05-05T12:04:18.538601801Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:04:18.540536208Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:04:29.357811202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:04:29.373581536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40638"
level=debug ts=2026-05-05T12:04:59.301219787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:04:59.360453595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:05:10.219288164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37102"
level=error ts=2026-05-05T12:05:24.64785017Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:05:24.649853691Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:05:24.651597952Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:05:24.651627723Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:05:29.363502098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T12:05:56.469649717Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:05:59.366273015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:06:29.369682638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T12:06:56.802664601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:06:59.37273368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:07:10.241011035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:47492"
ts=2026-05-05T12:07:29.376082928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:07:29.388684041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:54200"
level=debug ts=2026-05-05T12:07:34.599002894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:07:59.379479705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:08:10.245901546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:60654"
level=debug ts=2026-05-05T12:08:21.863815189Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:08:21.86545759Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:08:29.382871183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:08:59.385889859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:08:59.396694049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:32956"
level=debug ts=2026-05-05T12:09:23.048379746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:09:29.389135713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:09:29.400779203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42696"
ts=2026-05-05T12:09:40.251355934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:54226"
ts=2026-05-05T12:09:59.392785433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:09:59.406326588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33404"
level=debug ts=2026-05-05T12:10:04.131094083Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:10:29.397075793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:10:29.409553321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:44980"
ts=2026-05-05T12:10:33.994847307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55352"
ts=2026-05-05T12:10:40.255133397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:53274"
level=debug ts=2026-05-05T12:10:44.796342023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:10:59.40052347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:11:29.404757088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:11:29.415693635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41912"
ts=2026-05-05T12:11:33.999567366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37432"
level=debug ts=2026-05-05T12:11:39.923824403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:11:59.412955545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:12:29.416247181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:12:29.420495758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:44988"
level=debug ts=2026-05-05T12:12:39.91093592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:12:39.912841381Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:12:40.263259706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:54438"
ts=2026-05-05T12:12:59.418872274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:13:10.266437033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:60256"
ts=2026-05-05T12:13:29.42192899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:13:29.425407905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:38326"
level=debug ts=2026-05-05T12:13:33.035391301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:13:40.26932797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:48952"
ts=2026-05-05T12:13:59.424987984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:13:59.429007465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43282"
level=debug ts=2026-05-05T12:14:17.360576784Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:14:29.428244977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:14:40.274411277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:35964"
ts=2026-05-05T12:14:59.430992429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T12:15:09.052292501Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:15:24.647286429Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:15:24.649237568Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:15:24.651133565Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:15:24.65115697Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:15:29.433978193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:15:29.435764544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:48194"
level=debug ts=2026-05-05T12:15:46.95728524Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:15:59.436746707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:15:59.43910515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53104"
ts=2026-05-05T12:16:04.016913437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35074"
ts=2026-05-05T12:16:10.279996281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:34240"
ts=2026-05-05T12:16:29.439681142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
level=debug ts=2026-05-05T12:16:29.773997665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:16:29.77593494Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:16:34.020244644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55730"
ts=2026-05-05T12:16:40.282355966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:52234"
ts=2026-05-05T12:16:59.442544641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:17:29.445375388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
level=debug ts=2026-05-05T12:17:30.63356651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:17:59.44742352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58640"
ts=2026-05-05T12:17:59.448023208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:18:04.027227003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41712"
level=debug ts=2026-05-05T12:18:29.062930217Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:18:29.451358116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:18:59.453960144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:19:04.032182967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:45408"
ts=2026-05-05T12:19:10.293158686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:60724"
level=debug ts=2026-05-05T12:19:22.486807085Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:19:29.454618289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46016"
ts=2026-05-05T12:19:29.45699046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:19:59.459801153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T12:20:00.089484173Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:20:04.037078414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56090"
ts=2026-05-05T12:20:29.463395283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:20:34.040130471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56828"
ts=2026-05-05T12:20:40.301026285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:56200"
level=debug ts=2026-05-05T12:20:40.315328907Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:20:40.316865685Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:20:59.466179699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:21:29.469205935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
level=debug ts=2026-05-05T12:21:30.455622891Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:21:34.045081381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:36672"
ts=2026-05-05T12:21:59.472193154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T12:22:10.465940061Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:22:29.468881545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43986"
ts=2026-05-05T12:22:29.475215792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:22:59.477882891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:23:04.052250825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53662"
level=debug ts=2026-05-05T12:23:06.305556089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:23:10.311677498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:51138"
ts=2026-05-05T12:23:29.480545656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:23:34.055568559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57778"
ts=2026-05-05T12:23:40.314338229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:48300"
ts=2026-05-05T12:23:59.483982988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T12:24:06.917425606Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:24:29.486591045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:24:34.059344637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:36372"
level=debug ts=2026-05-05T12:24:42.23003138Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:24:42.231597097Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:24:59.490174757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T12:25:24.101662767Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:25:24.645979826Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:25:24.647827443Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:25:24.649611302Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:25:24.649638189Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:25:29.492739004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:25:34.06399531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56864"
ts=2026-05-05T12:25:59.485490445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46306"
ts=2026-05-05T12:25:59.496176538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:26:10.324051574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38986"
level=debug ts=2026-05-05T12:26:23.181623423Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:26:29.499197185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:26:59.502357071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T12:27:00.473385579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:27:29.493089305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58496"
ts=2026-05-05T12:27:29.505874939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:27:40.328931414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37224"
ts=2026-05-05T12:27:59.496910406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:38154"
ts=2026-05-05T12:27:59.508578895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
level=debug ts=2026-05-05T12:28:02.481434826Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:28:04.074221348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50680"
ts=2026-05-05T12:28:29.499811258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47774"
ts=2026-05-05T12:28:29.511415695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:28:34.077466827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35726"
ts=2026-05-05T12:28:40.33411075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:33788"
level=debug ts=2026-05-05T12:28:46.528711968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:28:46.530483681Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:28:59.51473935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T12:29:28.669451376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:29:29.50486481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46096"
ts=2026-05-05T12:29:29.517500674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:29:59.520601987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T12:30:21.181107979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:30:29.510131973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47924"
ts=2026-05-05T12:30:29.523558748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:30:34.084313163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51718"
level=debug ts=2026-05-05T12:30:55.761745131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:30:59.51330146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:38798"
ts=2026-05-05T12:30:59.526744777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:31:29.517026839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42488"
ts=2026-05-05T12:31:29.529486858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:31:34.08892401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:36290"
level=debug ts=2026-05-05T12:31:50.363785697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:31:59.520608488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59838"
ts=2026-05-05T12:31:59.532029309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:32:29.534842133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:32:34.09361558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48550"
level=debug ts=2026-05-05T12:32:37.958754162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:32:37.96027483Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:32:59.526092904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46124"
ts=2026-05-05T12:32:59.537582423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:33:04.096509561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:34122"
level=debug ts=2026-05-05T12:33:25.004956362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:33:29.540941327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:33:40.351942041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:60730"
ts=2026-05-05T12:33:59.544175026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
level=debug ts=2026-05-05T12:34:21.347697152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:34:29.533839904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51226"
ts=2026-05-05T12:34:29.547304451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:34:59.53739989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34762"
ts=2026-05-05T12:34:59.549959014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T12:35:16.869287225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:35:24.646006545Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:35:24.647962904Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:35:24.649879493Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:35:24.649909454Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:35:29.540887073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52930"
ts=2026-05-05T12:35:29.553252588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:35:59.556107345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T12:36:10.390643547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:36:29.559059496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
level=debug ts=2026-05-05T12:36:54.403396592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:36:54.405452038Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:36:59.562291771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:37:29.56589462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T12:37:43.165871228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:37:59.552504329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:38832"
ts=2026-05-05T12:37:59.569353611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:38:29.572692471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T12:38:39.000519022Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:38:40.373011489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:53488"
ts=2026-05-05T12:38:59.575570339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T12:39:16.624945823Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:39:29.578632371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:39:40.378015026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:49844"
ts=2026-05-05T12:39:59.581531604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T12:40:08.375002546Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:40:10.380389073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38122"
ts=2026-05-05T12:40:29.584413896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:40:40.383079671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37104"
ts=2026-05-05T12:40:59.565075839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:60704"
ts=2026-05-05T12:40:59.587788847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:41:04.125417767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33030"
level=debug ts=2026-05-05T12:41:07.975413155Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:41:07.977171229Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:41:29.568627657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:54020"
ts=2026-05-05T12:41:29.590613652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:41:59.593856642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T12:42:02.131814618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:42:10.38909836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:44612"
ts=2026-05-05T12:42:29.596533843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:42:34.132428638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42454"
level=debug ts=2026-05-05T12:42:49.217546555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:42:59.599844717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:43:04.135845357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42106"
ts=2026-05-05T12:43:10.393179078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:35010"
ts=2026-05-05T12:43:29.602350294Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T12:43:39.931968553Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:43:59.579763747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51606"
ts=2026-05-05T12:43:59.605239989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:44:29.582747589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49550"
ts=2026-05-05T12:44:29.60831944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T12:44:35.009290592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:44:40.39942361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:48946"
ts=2026-05-05T12:44:59.610844224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
level=debug ts=2026-05-05T12:45:14.828155956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:45:14.829905962Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:45:24.646167966Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:45:24.648206003Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:45:24.649996704Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:45:24.65001818Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:45:29.587553258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33338"
ts=2026-05-05T12:45:29.613962594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:45:59.616817324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T12:46:10.201851336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:46:29.592974329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39604"
ts=2026-05-05T12:46:29.619444523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:46:40.406961861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:54098"
ts=2026-05-05T12:46:59.596688702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36888"
ts=2026-05-05T12:46:59.622297599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T12:47:00.391641112Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:47:29.625442672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T12:47:36.073131611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:47:59.628208478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:48:04.156554334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:49544"
level=debug ts=2026-05-05T12:48:16.964799089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:48:29.631602409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:48:59.634932392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:49:04.16144574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55470"
level=debug ts=2026-05-05T12:49:04.442088808Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:49:04.443865699Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:49:29.637957576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:49:34.16453325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:60632"
level=debug ts=2026-05-05T12:49:39.299247988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:49:59.610011948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45362"
ts=2026-05-05T12:49:59.640692367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T12:50:14.54725691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:50:29.612930997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51484"
ts=2026-05-05T12:50:29.643397793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:50:34.169714005Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53652"
ts=2026-05-05T12:50:59.615575594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:50534"
ts=2026-05-05T12:50:59.646370567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T12:51:08.698845588Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:51:10.424214178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:41964"
ts=2026-05-05T12:51:29.618664128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41700"
ts=2026-05-05T12:51:29.649338531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:51:40.426650962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:48204"
level=debug ts=2026-05-05T12:51:50.257357872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:51:59.652286764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:52:29.655646414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
level=debug ts=2026-05-05T12:52:42.22268979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:52:42.224577514Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:52:59.658809086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:53:29.628063342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:38852"
ts=2026-05-05T12:53:29.661807681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T12:53:31.585548013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:53:40.43364351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37196"
ts=2026-05-05T12:53:59.630911424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36502"
ts=2026-05-05T12:53:59.664516561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T12:54:09.822249641Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:54:29.66739045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:54:34.182822461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:38016"
ts=2026-05-05T12:54:40.438008136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:55094"
ts=2026-05-05T12:54:59.635881303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53230"
ts=2026-05-05T12:54:59.670527489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T12:55:06.739055821Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:55:24.646813138Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:55:24.649096765Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:55:24.651195108Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T12:55:24.651215229Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:55:29.638743847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:38430"
ts=2026-05-05T12:55:29.673212886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T12:55:34.18764301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35488"
ts=2026-05-05T12:55:40.443375849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:44816"
level=debug ts=2026-05-05T12:55:44.531579623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:55:59.675849111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:56:04.190622427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53876"
ts=2026-05-05T12:56:29.64453877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52000"
ts=2026-05-05T12:56:29.678326741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:56:40.448439412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:55034"
level=debug ts=2026-05-05T12:56:43.661578006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:56:43.663145708Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T12:56:59.648061889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:37032"
ts=2026-05-05T12:56:59.681944106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T12:57:29.684829532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T12:57:30.906763333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:57:34.197315833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40586"
ts=2026-05-05T12:57:59.653431264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49608"
ts=2026-05-05T12:57:59.687987018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:58:04.200225383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52326"
level=debug ts=2026-05-05T12:58:19.461629082Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:58:29.690400318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T12:58:59.693025048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T12:59:02.161129324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:59:04.203977817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51980"
ts=2026-05-05T12:59:29.695917388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T12:59:45.746023095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:59:59.699218396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T13:00:29.701789439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T13:00:39.928552623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T13:00:39.930355924Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T13:00:59.704324679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T13:01:10.465051588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:44820"
level=debug ts=2026-05-05T13:01:16.606668165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T13:01:29.707174171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
level=debug ts=2026-05-05T13:01:58.238636917Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T13:01:59.709856021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T13:02:29.712797369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T13:02:40.471963208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:46890"
level=debug ts=2026-05-05T13:02:48.799812082Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T13:02:59.71534043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T13:03:04.220029359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54410"
ts=2026-05-05T13:03:29.717855539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
level=debug ts=2026-05-05T13:03:49.052185011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T13:03:59.681482437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42338"
ts=2026-05-05T13:03:59.720968222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-2pb6t-87d8b308 10.128.0.86:7946"
ts=2026-05-05T13:04:10.478943648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:51714"
ts=2026-05-05T13:04:29.724210913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
level=debug ts=2026-05-05T13:04:50.096455092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T13:04:50.098178019Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T13:04:59.727589529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T13:05:04.229114642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50590"
level=error ts=2026-05-05T13:05:24.648401056Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T13:05:24.650353923Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T13:05:24.65211602Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T13:05:24.652139984Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T13:05:29.225060221Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T13:05:29.730789513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T13:05:59.691585582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:55384"
ts=2026-05-05T13:05:59.734049427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmt6d-ba91a831 10.128.0.83:7946"
ts=2026-05-05T13:06:04.233907809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:38474"
ts=2026-05-05T13:06:10.486913891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:59304"
level=debug ts=2026-05-05T13:06:25.336103094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T13:06:29.737063059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
ts=2026-05-05T13:06:59.740131535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ca84f 10.130.0.69:7946"
level=debug ts=2026-05-05T13:07:04.330975881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T13:07:10.490940592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:41036"