level=info ts=2026-05-11T11:15:56.650712863Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-11T11:15:56.65103017Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-11T11:15:56.652258401Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-11T11:15:56.6525357Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-11T11:15:56.652599265Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.82 inf=eth0
level=debug ts=2026-05-11T11:15:56.653328599Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-11T11:15:56.653391127Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.82 inf=eth0
level=info ts=2026-05-11T11:15:56.653473913Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-dbd70d4e
ts=2026-05-11T11:15:56.653539291Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-11T11:15:56.653552642Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.82 advertisePort=7946
level=debug ts=2026-05-11T11:15:56.653570187Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.82 advertisePort=7946
level=info ts=2026-05-11T11:15:56.653687197Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-11T11:15:56.660462248Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-11T11:15:56.660478617Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-11T11:15:56.660488041Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.803343ms
level=error ts=2026-05-11T11:15:56.660496249Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-11T11:15:56.660506632Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-11T11:15:56.662748462Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T11:15:56.662767575Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-11T11:15:56.670262764Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-11T11:15:56.67027654Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-11T11:15:56.670293979Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-05-11T11:15:56.670303109Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-11T11:15:56.6703147Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-11T11:15:56.670328286Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-11T11:15:56.670416708Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-11T11:15:56.670528073Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-11T11:15:56.67061957Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-11T11:15:56.670629295Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-11T11:15:56.670690444Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-11T11:15:56.670819471Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-11T11:15:56.670871039Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-11T11:15:56.671136533Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-11T11:15:56.671155033Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-11T11:15:56.671190755Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-11T11:15:57.671312574Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-11T11:15:57.67133413Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-11T11:15:57.671357209Z caller=loki.go:615 msg="Loki started" startup_time=1.080026515s
ts=2026-05-11T11:15:58.454282096Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T11:15:58.454306229Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:16:01.969320699Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T11:16:01.969343597Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-11T11:16:02.672361346Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-11T11:16:02.672409685Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-11T11:16:06.23018275Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T11:16:06.230204748Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-11T11:16:12.106329035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:16:14.728477853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.73:7946"
ts=2026-05-11T11:16:14.730129658Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.77:7946: dial tcp 10.130.0.77:7946: connect: connection refused"
ts=2026-05-11T11:16:14.73095442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.72:7946"
ts=2026-05-11T11:16:14.732359603Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.75:7946: dial tcp 10.130.0.75:7946: connect: connection refused"
ts=2026-05-11T11:16:14.732434541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.82:7946"
ts=2026-05-11T11:16:14.732470616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:43150"
ts=2026-05-11T11:16:14.734661445Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.84:7946: dial tcp 10.128.0.84:7946: connect: connection refused"
level=info ts=2026-05-11T11:16:14.734671383Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=18.074160478s
ts=2026-05-11T11:16:17.787902734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:51772"
ts=2026-05-11T11:16:17.866242807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:37506"
ts=2026-05-11T11:16:22.168066095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:34824"
level=debug ts=2026-05-11T11:16:24.209020747Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:16:32.595956797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T11:16:36.048096501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:34210"
level=debug ts=2026-05-11T11:16:45.198044194Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:16:45.223462844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:48994"
ts=2026-05-11T11:17:02.597747823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
level=debug ts=2026-05-11T11:17:05.3979897Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:17:32.599986823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:17:36.051501123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:57522"
level=debug ts=2026-05-11T11:17:36.900625598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-11T11:17:36.902235741Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:18:02.602172295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:18:06.053046463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:58502"
level=debug ts=2026-05-11T11:18:11.372002401Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:18:16.026489277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:56934"
ts=2026-05-11T11:18:32.604127714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:18:45.230407016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43494"
ts=2026-05-11T11:18:46.028552269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:50548"
ts=2026-05-11T11:19:02.605729665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T11:19:15.23203212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:34180"
ts=2026-05-11T11:19:16.029966432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:40414"
level=debug ts=2026-05-11T11:19:18.798277703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:19:32.607602303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:19:45.234496411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43318"
ts=2026-05-11T11:20:02.609147467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:20:16.033193653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:46844"
ts=2026-05-11T11:20:32.610828582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
level=debug ts=2026-05-11T11:20:35.926583025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:20:36.059624324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:59968"
ts=2026-05-11T11:20:45.237214549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43130"
ts=2026-05-11T11:20:46.035326061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:46572"
ts=2026-05-11T11:21:02.612699488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:21:15.238978132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:46498"
ts=2026-05-11T11:21:16.036715848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:47028"
ts=2026-05-11T11:21:32.614356779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
level=debug ts=2026-05-11T11:21:41.565173154Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:22:02.616278324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:22:16.040231756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:42452"
ts=2026-05-11T11:22:32.618097527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T11:22:45.243128739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52160"
level=debug ts=2026-05-11T11:22:49.104391162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-11T11:22:49.105999556Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:23:02.620058896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:23:32.621890385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
level=debug ts=2026-05-11T11:23:44.92197712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:24:02.623884713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:24:15.248459023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:47148"
ts=2026-05-11T11:24:32.626129807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T11:24:36.073150345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:37828"
level=debug ts=2026-05-11T11:24:43.491388197Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:24:45.249895629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:47208"
ts=2026-05-11T11:25:02.628297194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
level=debug ts=2026-05-11T11:25:32.438465199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:25:32.630549766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:26:02.632706606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
level=info ts=2026-05-11T11:26:02.672937126Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-11T11:26:02.677989649Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:26:02.679794675Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:26:02.681347623Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:26:02.681372329Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-11T11:26:02.681384557Z caller=tables_manager.go:148 msg="compactor started"
level=debug ts=2026-05-11T11:26:13.56792633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:26:32.634917088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
level=debug ts=2026-05-11T11:26:59.918663772Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T11:26:59.920317925Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:27:02.640821818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:27:16.054310294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:56946"
ts=2026-05-11T11:27:32.642587838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:27:45.259474985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38640"
level=debug ts=2026-05-11T11:27:48.579852076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:28:02.64441134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T11:28:16.05743328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:59360"
ts=2026-05-11T11:28:32.646390621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:28:46.059037082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:54348"
level=debug ts=2026-05-11T11:28:50.247356549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:29:02.647965597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:29:15.265023511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38744"
ts=2026-05-11T11:29:32.649722852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
level=debug ts=2026-05-11T11:29:36.130900058Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:29:45.266673341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:49222"
ts=2026-05-11T11:30:02.651147478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
level=debug ts=2026-05-11T11:30:21.562584055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:30:32.652906418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:30:45.269980059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41446"
ts=2026-05-11T11:30:46.065653362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:51662"
ts=2026-05-11T11:31:02.655058897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
level=debug ts=2026-05-11T11:31:05.141053289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T11:31:05.142849361Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:31:15.272460405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:39002"
ts=2026-05-11T11:31:32.656630712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:31:36.090814406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:53868"
level=debug ts=2026-05-11T11:31:54.634332832Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:32:02.658361885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:32:15.275297221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53870"
ts=2026-05-11T11:32:32.660314283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
level=debug ts=2026-05-11T11:32:39.872477199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:32:45.27768349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52224"
ts=2026-05-11T11:33:02.661980485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:33:06.095798009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:38460"
ts=2026-05-11T11:33:32.664213105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
level=debug ts=2026-05-11T11:33:39.997454894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:33:45.280705723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53348"
ts=2026-05-11T11:34:02.665633842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:34:15.282812142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42194"
ts=2026-05-11T11:34:16.075462894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:55134"
level=debug ts=2026-05-11T11:34:18.23870741Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:34:32.667368837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:34:46.077709076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:52250"
ts=2026-05-11T11:35:02.669603289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
level=debug ts=2026-05-11T11:35:06.638018345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T11:35:06.639824191Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:35:32.671885196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:35:45.287817046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:48558"
level=debug ts=2026-05-11T11:36:01.589454335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:36:02.675877375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
level=error ts=2026-05-11T11:36:02.689066395Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:36:02.691119264Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:36:02.693231248Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:36:02.693253885Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:36:15.289501053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:56424"
ts=2026-05-11T11:36:16.081816956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39800"
ts=2026-05-11T11:36:32.681421737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:36:36.106542596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:57560"
level=debug ts=2026-05-11T11:36:40.180242976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:36:45.291270853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:44044"
ts=2026-05-11T11:36:46.083657128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:52374"
ts=2026-05-11T11:37:02.682977898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:37:06.108634725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:50434"
ts=2026-05-11T11:37:15.293335573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42642"
level=debug ts=2026-05-11T11:37:22.388424674Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:37:32.685060772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:37:45.295167992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43380"
ts=2026-05-11T11:37:46.087376677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:42526"
ts=2026-05-11T11:38:02.686603474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T11:38:16.08936476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:34550"
level=debug ts=2026-05-11T11:38:20.503330974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:38:32.688235236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:38:46.091126221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:55382"
ts=2026-05-11T11:39:02.690628532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
level=debug ts=2026-05-11T11:39:03.275138507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T11:39:03.276761659Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:39:16.09328957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:44124"
ts=2026-05-11T11:39:32.69227995Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:39:36.115686389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:38420"
level=debug ts=2026-05-11T11:39:40.992255479Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:40:02.694363891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T11:40:06.117660186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:58650"
ts=2026-05-11T11:40:15.303900702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53770"
ts=2026-05-11T11:40:32.696421335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
level=debug ts=2026-05-11T11:40:35.770752031Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:40:45.30586952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:55622"
ts=2026-05-11T11:41:02.698112011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:41:15.307453789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42686"
level=debug ts=2026-05-11T11:41:19.590933304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:41:32.700017345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:41:45.308910775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:35114"
ts=2026-05-11T11:42:02.701805088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:42:06.123445956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:58624"
level=debug ts=2026-05-11T11:42:21.218759132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:42:32.704100653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:43:02.705513255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
level=debug ts=2026-05-11T11:43:11.238340012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T11:43:11.239939926Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:43:32.707645898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T11:43:46.104348084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:58170"
level=debug ts=2026-05-11T11:43:56.735323982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:44:02.709820493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:44:15.316594261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37122"
ts=2026-05-11T11:44:32.712000329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
level=debug ts=2026-05-11T11:44:52.268639453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:45:02.713558178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:45:15.31973758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:48144"
ts=2026-05-11T11:45:32.71544424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:45:36.133888226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:48656"
level=debug ts=2026-05-11T11:45:39.510697513Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T11:46:02.686443179Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:46:02.688835478Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:46:02.691014397Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:46:02.691043368Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:46:02.71766489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:46:16.110838564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:37382"
level=debug ts=2026-05-11T11:46:25.946824051Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:46:32.719429275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T11:46:36.137748306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:42576"
ts=2026-05-11T11:46:45.324084663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42464"
ts=2026-05-11T11:46:46.112723306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:53030"
ts=2026-05-11T11:47:02.721458484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
level=debug ts=2026-05-11T11:47:03.775218405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T11:47:03.776912901Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:47:06.139897136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:57124"
ts=2026-05-11T11:47:32.722916573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:47:45.328456978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:48514"
level=debug ts=2026-05-11T11:48:00.824474334Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:48:02.724553399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T11:48:32.72625837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:48:36.144930727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:33902"
level=debug ts=2026-05-11T11:49:01.787285125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:49:02.728510184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T11:49:16.121058451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:59062"
ts=2026-05-11T11:49:32.730481641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:49:45.333731031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:59630"
level=debug ts=2026-05-11T11:50:01.34889436Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:50:02.732673087Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:50:32.734130536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:50:36.150717493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:36600"
ts=2026-05-11T11:50:45.337329845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43590"
level=debug ts=2026-05-11T11:51:00.083231356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:51:02.735433764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:51:16.127003458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:55846"
ts=2026-05-11T11:51:32.737557998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:51:36.153886403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:34866"
level=debug ts=2026-05-11T11:51:56.411046188Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T11:51:56.412555845Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:52:02.73911241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:52:16.129685623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:53232"
level=debug ts=2026-05-11T11:52:32.042555489Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:52:32.740852292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T11:53:02.742760092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
level=debug ts=2026-05-11T11:53:11.606657704Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:53:15.344750665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:36898"
ts=2026-05-11T11:53:32.744504624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T11:53:45.347019283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:40864"
level=debug ts=2026-05-11T11:53:50.444955453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:54:02.745931146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:54:16.134584772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:37666"
ts=2026-05-11T11:54:32.75334735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
level=debug ts=2026-05-11T11:54:39.108586981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:54:46.136877797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:32872"
ts=2026-05-11T11:55:02.755107568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
level=debug ts=2026-05-11T11:55:16.144185416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T11:55:16.145953619Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:55:32.75669329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
level=error ts=2026-05-11T11:56:02.686952088Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:56:02.688999454Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:56:02.690745819Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:56:02.690766252Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:56:02.758599501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:56:06.167951284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:51876"
level=debug ts=2026-05-11T11:56:11.82882302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:56:15.35336301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53192"
ts=2026-05-11T11:56:16.142169157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:47312"
ts=2026-05-11T11:56:32.760732684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T11:56:46.144154976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:42574"
ts=2026-05-11T11:57:02.762894797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
level=debug ts=2026-05-11T11:57:10.465166676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:57:16.146162437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:51988"
ts=2026-05-11T11:57:32.764997186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T11:57:36.172498523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:34592"
ts=2026-05-11T11:58:02.767221608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T11:58:06.174456006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:38132"
level=debug ts=2026-05-11T11:58:08.951496006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:58:15.360049047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53960"
ts=2026-05-11T11:58:32.768795853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T11:58:46.151553354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:37798"
ts=2026-05-11T11:59:02.770451422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
level=debug ts=2026-05-11T11:59:04.234238856Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:59:16.153943935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:50384"
ts=2026-05-11T11:59:32.772530707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
level=debug ts=2026-05-11T11:59:56.162602281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T11:59:56.164212779Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:00:02.774870653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T12:00:16.156885424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:60010"
level=debug ts=2026-05-11T12:00:30.361653609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:00:32.777257574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T12:01:02.779607618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T12:01:06.182287688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:56034"
ts=2026-05-11T12:01:16.160681864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:47166"
level=debug ts=2026-05-11T12:01:26.06967953Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:01:32.782260663Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T12:01:36.183727253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:58044"
ts=2026-05-11T12:02:02.784536574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
level=debug ts=2026-05-11T12:02:13.806446587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:02:32.786040827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T12:02:46.1660098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:40444"
level=debug ts=2026-05-11T12:03:00.835917313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:03:02.787703927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6acc0d33 10.130.0.76:7946"
ts=2026-05-11T12:03:32.789434228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
level=debug ts=2026-05-11T12:03:43.909848143Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:03:43.911508817Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:04:02.791622855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
level=debug ts=2026-05-11T12:04:19.220654144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:04:32.793190782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T12:04:46.172306003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:40000"
level=debug ts=2026-05-11T12:05:00.190289945Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:05:02.794500316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
ts=2026-05-11T12:05:06.193277064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:44148"
ts=2026-05-11T12:05:15.381818759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:60998"
ts=2026-05-11T12:05:16.173726168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:54942"
ts=2026-05-11T12:05:32.796308277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xqrdd-2ee7fc0b 10.130.0.73:7946"
level=debug ts=2026-05-11T12:05:42.365872304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:06:02.684747209Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:06:02.686575034Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:06:02.688560862Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:06:02.688584146Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:06:02.798431251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-7m2v4-0528f113 10.130.0.72:7946"
ts=2026-05-11T12:06:16.17756139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:59686"