level=info ts=2026-04-29T19:18:14.394924408Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-04-29T19:18:14.394960035Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-29T19:18:14.397586079Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-29T19:18:14.398013827Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-29T19:18:14.398111629Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.81 inf=eth0
level=debug ts=2026-04-29T19:18:14.398740795Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-29T19:18:14.39882884Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.81 inf=eth0
level=info ts=2026-04-29T19:18:14.398830247Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-a35146bd
ts=2026-04-29T19:18:14.398902069Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-29T19:18:14.398913837Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.81 advertisePort=7946
level=debug ts=2026-04-29T19:18:14.39892886Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.81 advertisePort=7946
level=info ts=2026-04-29T19:18:14.399055858Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-29T19:18:14.405414759Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-29T19:18:14.405436618Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-29T19:18:14.405451958Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.397773ms
level=error ts=2026-04-29T19:18:14.405464004Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-04-29T19:18:14.405480054Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-29T19:18:14.408959204Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T19:18:14.408974554Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-29T19:18:14.416904755Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-29T19:18:14.416914567Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-29T19:18:14.416905801Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-29T19:18:14.416949379Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-29T19:18:14.416999594Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-29T19:18:14.417015348Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-29T19:18:14.417033043Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-29T19:18:14.417335256Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-29T19:18:14.41734652Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-29T19:18:14.417351945Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-29T19:18:14.417413883Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-29T19:18:14.417412541Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-29T19:18:14.417440915Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-29T19:18:14.417596523Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-29T19:18:14.417614409Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-29T19:18:14.417655649Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-29T19:18:15.417799912Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-29T19:18:15.61536435Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-29T19:18:15.615416587Z caller=loki.go:615 msg="Loki started" startup_time=1.292351931s
ts=2026-04-29T19:18:16.152755752Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T19:18:16.152784304Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T19:18:19.097453992Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T19:18:19.097483569Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-29T19:18:20.615923129Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-29T19:18:20.615996018Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-29T19:18:26.680065247Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T19:18:26.680091309Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-29T19:18:28.932577872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T19:18:36.356120655Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T19:18:36.356146643Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T19:18:40.882631275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54198"
level=debug ts=2026-04-29T19:18:45.851933917Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T19:18:53.519263716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:43032"
ts=2026-04-29T19:18:54.580629995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39462"
ts=2026-04-29T19:18:55.538391131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.63:7946"
ts=2026-04-29T19:18:55.539697035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.81:7946"
ts=2026-04-29T19:18:55.539800832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:51262"
ts=2026-04-29T19:18:55.540299767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.79:7946"
ts=2026-04-29T19:18:55.541591916Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.78:7946: dial tcp 10.128.0.78:7946: connect: connection refused"
ts=2026-04-29T19:18:55.542237105Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.80:7946: dial tcp 10.128.0.80:7946: connect: connection refused"
ts=2026-04-29T19:18:55.543140144Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.78:7946: dial tcp 10.130.0.78:7946: connect: connection refused"
ts=2026-04-29T19:18:55.543664371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.79:7946"
level=info ts=2026-04-29T19:18:55.544470688Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=41.138983886s
ts=2026-04-29T19:19:06.598394074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T19:19:08.267653114Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T19:19:15.886965729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:53502"
level=debug ts=2026-04-29T19:19:32.67378686Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T19:19:36.600614257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:19:45.890118083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:43204"
ts=2026-04-29T19:19:59.801129663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47160"
level=debug ts=2026-04-29T19:20:01.200194621Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-29T19:20:01.201932988Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T19:20:06.602598914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:20:15.893371121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:52282"
ts=2026-04-29T19:20:36.605040041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T19:20:36.819368503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T19:21:06.607733489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T19:21:27.003474134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T19:21:36.609936794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T19:22:06.61100633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T19:22:15.90377077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:39882"
level=debug ts=2026-04-29T19:22:29.099935542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T19:22:36.612694497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:22:45.906609969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:59674"
ts=2026-04-29T19:22:59.814107654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35716"
ts=2026-04-29T19:23:06.614961957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:23:15.909774172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:54618"
level=debug ts=2026-04-29T19:23:17.576667738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T19:23:29.815877938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:32874"
ts=2026-04-29T19:23:36.61780172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:23:45.912917228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:41334"
ts=2026-04-29T19:23:59.817791257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:59854"
ts=2026-04-29T19:24:06.618980808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T19:24:15.91512384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:43958"
level=debug ts=2026-04-29T19:24:25.210280933Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-29T19:24:25.212230135Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T19:24:36.621027542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:25:06.623198379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:25:15.921307644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:56652"
level=debug ts=2026-04-29T19:25:24.239286926Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:25:36.625133397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T19:25:43.669550308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56382"
ts=2026-04-29T19:25:45.923590946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:42042"
ts=2026-04-29T19:26:06.627183863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T19:26:24.000428074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:26:29.831628023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:49420"
ts=2026-04-29T19:26:36.629601304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:26:59.833687332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54982"
ts=2026-04-29T19:27:06.632055643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:27:13.676386653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34860"
level=debug ts=2026-04-29T19:27:15.310396814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:27:15.931931811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:34844"
ts=2026-04-29T19:27:29.836080868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:50612"
ts=2026-04-29T19:27:36.634680308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:27:45.934763127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:57234"
level=debug ts=2026-04-29T19:28:04.18495797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:28:06.637297262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:28:15.937447113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:34750"
level=info ts=2026-04-29T19:28:20.61650925Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-29T19:28:20.620065052Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T19:28:20.621969524Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T19:28:20.623879741Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T19:28:20.623904192Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-29T19:28:20.623916261Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-04-29T19:28:36.639808622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T19:28:43.682796687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59286"
ts=2026-04-29T19:28:45.939700411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:42050"
level=debug ts=2026-04-29T19:28:52.312544144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T19:28:52.314097212Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T19:29:06.641641234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:29:29.844499011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:51154"
level=debug ts=2026-04-29T19:29:29.936476848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:29:36.643644975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:30:06.645780648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T19:30:07.207172551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:30:13.689356971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53936"
ts=2026-04-29T19:30:15.948262187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:46202"
ts=2026-04-29T19:30:29.850495207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:34220"
ts=2026-04-29T19:30:36.648265716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:30:43.690361563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59406"
level=debug ts=2026-04-29T19:31:05.257668443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:31:06.650529489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:31:36.653339955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:31:45.957507968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:33850"
ts=2026-04-29T19:31:59.857120997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:38650"
ts=2026-04-29T19:32:06.656159257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T19:32:06.711360657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:32:15.96024519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:37792"
ts=2026-04-29T19:32:36.657989921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:32:52.244689327Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T19:32:52.246591426Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T19:33:06.659846317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:33:36.662851907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T19:33:37.50482664Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:33:59.867492117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48820"
ts=2026-04-29T19:34:06.666037185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:34:13.709781526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52636"
ts=2026-04-29T19:34:15.97156585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:54174"
level=debug ts=2026-04-29T19:34:28.146166153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:34:36.668637529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:34:43.711453268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:38938"
ts=2026-04-29T19:35:06.671526295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T19:35:18.59661227Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:35:29.874381739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:56750"
ts=2026-04-29T19:35:36.673426725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T19:36:06.675622773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:36:15.983964732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:52312"
level=debug ts=2026-04-29T19:36:16.161970694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:36:36.677618432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T19:36:45.986422332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:38214"
level=debug ts=2026-04-29T19:36:56.165332878Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T19:36:56.1671085Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T19:36:59.883621127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:60012"
ts=2026-04-29T19:37:06.679755581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:37:15.989704982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:34722"
ts=2026-04-29T19:37:36.682077799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:37:43.727078386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52560"
level=debug ts=2026-04-29T19:37:52.706226209Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:37:59.889653765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:56096"
ts=2026-04-29T19:38:06.684698257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:38:16.003380692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:46854"
level=error ts=2026-04-29T19:38:20.628796156Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T19:38:20.631135547Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T19:38:20.6331443Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T19:38:20.633167394Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T19:38:36.693650958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:38:46.006678088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:42906"
level=debug ts=2026-04-29T19:38:46.331171559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:39:06.698674613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:39:29.897662832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54770"
ts=2026-04-29T19:39:36.701777095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T19:39:39.747800101Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:39:59.89961544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:37712"
ts=2026-04-29T19:40:06.704662441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:40:16.018636374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:58944"
ts=2026-04-29T19:40:29.904671766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:33502"
ts=2026-04-29T19:40:36.707417314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T19:40:39.516141812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:40:43.744378265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42844"
ts=2026-04-29T19:40:46.021204351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:52068"
ts=2026-04-29T19:41:06.709545323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T19:41:13.745738897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52216"
level=debug ts=2026-04-29T19:41:29.546661171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T19:41:29.548503664Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T19:41:36.711594702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:41:43.747345963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54588"
ts=2026-04-29T19:42:06.715649137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T19:42:10.743993351Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:42:29.917664349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:52302"
ts=2026-04-29T19:42:36.719361649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T19:43:03.798993947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:43:06.72250027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:43:36.72568196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:43:59.92626664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:42538"
level=debug ts=2026-04-29T19:44:02.110675244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:44:06.728058226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T19:44:13.760738492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39374"
ts=2026-04-29T19:44:16.041248294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:56592"
ts=2026-04-29T19:44:29.929208195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:57184"
ts=2026-04-29T19:44:36.730252065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T19:44:41.606915059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:44:43.761819628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45946"
ts=2026-04-29T19:44:46.044101404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:55654"
ts=2026-04-29T19:45:06.732721101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T19:45:16.047384758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:44118"
level=debug ts=2026-04-29T19:45:31.9562932Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T19:45:31.95802515Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T19:45:36.7348483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:46:06.737430199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T19:46:09.772420141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:46:13.767269463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49022"
ts=2026-04-29T19:46:29.941709672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:52804"
ts=2026-04-29T19:46:36.740242929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T19:46:44.15321141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:47:06.742593029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:47:13.77200927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36774"
level=debug ts=2026-04-29T19:47:34.531601815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:47:36.744733772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T19:47:43.773168527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37694"
ts=2026-04-29T19:47:59.950247763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:56482"
ts=2026-04-29T19:48:06.747548553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:48:16.064333826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:59816"
level=error ts=2026-04-29T19:48:20.62808456Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T19:48:20.630068195Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T19:48:20.631947274Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T19:48:20.631974276Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T19:48:25.062302711Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:48:36.749926901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:48:59.955803808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:44020"
level=debug ts=2026-04-29T19:49:00.87067841Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T19:49:00.872355352Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T19:49:06.75423271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:49:16.069690455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:57412"
ts=2026-04-29T19:49:36.756333403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:49:43.782848026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55518"
level=debug ts=2026-04-29T19:49:52.894214275Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:50:06.759175491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:50:36.762026011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:50:42.142943763Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:51:06.763718954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:51:13.800396907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59060"
ts=2026-04-29T19:51:29.970593824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54374"
level=debug ts=2026-04-29T19:51:35.245359408Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:51:36.766509156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:51:43.805452151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58956"
ts=2026-04-29T19:51:46.084858524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:57732"
ts=2026-04-29T19:51:59.972379184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:52392"
ts=2026-04-29T19:52:06.768118267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:52:23.279623266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:52:36.770579966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:52:43.811381178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51170"
level=debug ts=2026-04-29T19:53:00.64369112Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T19:53:00.645235661Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T19:53:06.773895317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:53:36.776052976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:53:45.305156344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:54:06.777784116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:54:16.100544352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:58154"
ts=2026-04-29T19:54:29.98554783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:50388"
ts=2026-04-29T19:54:36.780827777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:54:46.103689687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:41484"
level=debug ts=2026-04-29T19:54:46.634865981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:54:59.990587908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:60722"
ts=2026-04-29T19:55:06.783818164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:55:13.823066412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52132"
ts=2026-04-29T19:55:36.785869966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:55:38.402866459Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:55:46.110274946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:34654"
ts=2026-04-29T19:56:06.791419775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:56:19.115093806Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:56:36.802001773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:57:00.002366679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:53786"
ts=2026-04-29T19:57:06.805554811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T19:57:11.583434125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T19:57:11.585109713Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T19:57:16.119181819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:60266"
ts=2026-04-29T19:57:30.005007192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:58166"
ts=2026-04-29T19:57:36.80715961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T19:57:46.122279134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:37106"
ts=2026-04-29T19:58:06.809701145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T19:58:13.11745196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:58:16.124631584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:59174"
level=error ts=2026-04-29T19:58:20.629198281Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T19:58:20.631248776Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T19:58:20.633078568Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T19:58:20.633102044Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T19:58:36.812435156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T19:58:46.1271797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:39774"
level=debug ts=2026-04-29T19:59:02.897327867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:59:06.814697137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T19:59:16.129927389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:41602"
ts=2026-04-29T19:59:36.816946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:00:03.934702106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:00:06.819176672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:00:36.821607429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:00:43.851773978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:57508"
ts=2026-04-29T20:00:46.137477746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:39808"
level=debug ts=2026-04-29T20:00:59.524927626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:01:06.824050513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:01:13.853610176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33094"
ts=2026-04-29T20:01:36.826794411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T20:01:45.797927645Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:01:45.799644663Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:02:00.033118266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:34480"
ts=2026-04-29T20:02:06.829415562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T20:02:34.589878055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:02:36.832545561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:03:06.83431909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:03:11.155784528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:03:13.86504236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56426"
ts=2026-04-29T20:03:36.835769704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:03:43.866185043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:60472"
ts=2026-04-29T20:04:00.04430949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:51434"
level=debug ts=2026-04-29T20:04:00.657854555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:04:06.837634502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:04:36.839686688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:05:00.789135605Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:05:06.841780138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:05:16.165635063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:37502"
ts=2026-04-29T20:05:30.052101236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:39362"
ts=2026-04-29T20:05:36.843375401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:05:43.87734882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:35594"
level=debug ts=2026-04-29T20:05:51.304972542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:05:51.306898807Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:06:00.054021838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35528"
ts=2026-04-29T20:06:06.846070721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:06:13.87859784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42800"
ts=2026-04-29T20:06:30.056775075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35064"
ts=2026-04-29T20:06:36.848202316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:06:38.336828146Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:07:06.85017694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:07:13.881706645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:57548"
ts=2026-04-29T20:07:16.179195936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:54520"
level=debug ts=2026-04-29T20:07:33.999595257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:07:36.85249866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:08:00.063612183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:40208"
ts=2026-04-29T20:08:06.854749561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:08:13.885331629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56678"
level=debug ts=2026-04-29T20:08:18.614240252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:08:20.626252669Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T20:08:20.628694389Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T20:08:20.630276246Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T20:08:20.630299617Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:08:30.065742174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:41864"
ts=2026-04-29T20:08:36.85697905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:08:46.188043359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:54504"
level=debug ts=2026-04-29T20:08:56.357790191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:09:06.858119998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:09:16.190585384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:51610"
ts=2026-04-29T20:09:36.859468292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:09:57.823507193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:09:57.824968666Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:10:06.861243498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:10:30.078381454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:42732"
ts=2026-04-29T20:10:36.863151011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:10:55.290732838Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:11:06.865363574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:11:30.084096761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:60562"
ts=2026-04-29T20:11:36.867363106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:11:43.901098992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46572"
level=debug ts=2026-04-29T20:11:56.38179339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:12:06.869781513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:12:13.902552641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54474"
ts=2026-04-29T20:12:16.209038732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:51184"
ts=2026-04-29T20:12:30.089334464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:37358"
ts=2026-04-29T20:12:36.872398613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T20:12:49.36940577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:13:00.092023291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:40002"
ts=2026-04-29T20:13:06.875256018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:13:16.213891066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:42408"
level=debug ts=2026-04-29T20:13:31.532412513Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:13:36.878284025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:14:06.880495676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:14:23.22934373Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:14:23.231176727Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:14:36.882899088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:14:43.913912576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36172"
ts=2026-04-29T20:15:06.885597661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:15:09.871403676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:15:13.915789818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44980"
ts=2026-04-29T20:15:36.887574299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:15:43.916940411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:60476"
ts=2026-04-29T20:15:46.228075337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:56402"
level=debug ts=2026-04-29T20:15:49.292841864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:16:06.889465927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:16:13.918627211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:35080"
ts=2026-04-29T20:16:36.891699152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T20:16:46.974892107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:17:06.894917401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:17:36.896817599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:17:40.864963861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:18:06.899035411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:18:13.926770594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:40552"
level=error ts=2026-04-29T20:18:20.628000081Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T20:18:20.629746269Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T20:18:20.631402991Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T20:18:20.631424217Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:18:36.901760248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T20:18:37.177340644Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:18:37.178903551Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:18:43.927594819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47856"
ts=2026-04-29T20:19:06.904233616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:19:14.746143897Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:19:36.905727701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:19:50.019869219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:20:00.128600762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43038"
ts=2026-04-29T20:20:06.907700292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:20:13.933481734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:57086"
ts=2026-04-29T20:20:16.255303871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:57054"
ts=2026-04-29T20:20:36.910163599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:20:46.258675369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:58970"
level=debug ts=2026-04-29T20:20:52.235912073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:21:00.133081408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:34006"
ts=2026-04-29T20:21:06.913247819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:21:13.938083828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55754"
ts=2026-04-29T20:21:30.135225343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:50244"
ts=2026-04-29T20:21:36.915928656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:21:45.240050196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:22:00.137669708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:57582"
ts=2026-04-29T20:22:06.919235779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:22:16.266634592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:47114"
ts=2026-04-29T20:22:36.921734066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:22:42.477333065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:22:42.478749849Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:22:43.945309214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43112"
ts=2026-04-29T20:23:06.923200194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:23:19.969383685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:23:36.924849152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:24:06.926927294Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:24:13.194074406Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:24:13.953834431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42892"
ts=2026-04-29T20:24:30.151297653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47946"
ts=2026-04-29T20:24:36.928866142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:24:46.281217856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:34994"
ts=2026-04-29T20:25:00.153715217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:59682"
ts=2026-04-29T20:25:06.931602581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T20:25:13.134970139Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:25:36.934487917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:25:46.286666736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:54380"
level=debug ts=2026-04-29T20:25:53.643238225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:26:06.936481036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:26:16.28946404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:42978"
ts=2026-04-29T20:26:30.160160269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:44346"
ts=2026-04-29T20:26:36.938905519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:26:46.292798013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:51664"
level=debug ts=2026-04-29T20:26:47.190878097Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:26:47.192486246Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:27:00.162065217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:55324"
ts=2026-04-29T20:27:06.941323562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:27:16.295398436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:46628"
ts=2026-04-29T20:27:36.943108938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:27:46.310438093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:28:06.944623786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=error ts=2026-04-29T20:28:20.627877664Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T20:28:20.629852189Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T20:28:20.631916268Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T20:28:20.631937846Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T20:28:30.681130285Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:28:36.94697156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:29:00.172707224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:58580"
level=debug ts=2026-04-29T20:29:04.663602198Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:29:06.948556833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:29:36.949584593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:30:05.424785146Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:30:06.95213504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:30:16.313169428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:41978"
ts=2026-04-29T20:30:30.179122188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:37004"
ts=2026-04-29T20:30:36.95575867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:30:41.573438636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:30:41.575061768Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:31:06.958115804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:31:36.960101991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:31:38.189043347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:32:06.96126909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:32:18.317463083Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:32:30.189764796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54442"
ts=2026-04-29T20:32:36.963277372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:32:46.328470592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:42918"
ts=2026-04-29T20:33:00.191690326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:46894"
level=debug ts=2026-04-29T20:33:02.183648439Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:33:06.965846318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:33:30.194493438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:34892"
ts=2026-04-29T20:33:36.968075835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:33:46.334653103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:56698"
level=debug ts=2026-04-29T20:33:54.329008779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:34:06.970555851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:34:30.200095897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:51650"
ts=2026-04-29T20:34:36.973705177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T20:34:37.768871417Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:34:37.770412019Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:35:00.202470376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:60822"
ts=2026-04-29T20:35:06.976954697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:35:14.010787913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:35622"
ts=2026-04-29T20:35:16.343057175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:37506"
level=debug ts=2026-04-29T20:35:21.165116925Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:35:36.978878679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:35:44.012588734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:57816"
ts=2026-04-29T20:35:46.345278307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:49608"
ts=2026-04-29T20:36:00.207375735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:53938"
ts=2026-04-29T20:36:06.980449614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:36:16.347649186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:59728"
level=debug ts=2026-04-29T20:36:17.418188012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:36:36.982620589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:36:44.016730855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44358"
ts=2026-04-29T20:36:46.350990243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:57084"
ts=2026-04-29T20:37:06.984724502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:37:14.018325056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33678"
level=debug ts=2026-04-29T20:37:15.991555611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:37:16.35335689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:50674"
ts=2026-04-29T20:37:36.986359992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:37:58.571532282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:38:06.989000718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=error ts=2026-04-29T20:38:20.630046348Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T20:38:20.631936508Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T20:38:20.634108055Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T20:38:20.634141332Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:38:30.221132935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54300"
ts=2026-04-29T20:38:36.992077967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:38:44.02538567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58964"
level=debug ts=2026-04-29T20:38:44.323881766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:38:44.325480299Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:38:46.362092861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:36452"
ts=2026-04-29T20:39:00.223575465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:53840"
ts=2026-04-29T20:39:06.993651472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:39:26.633923834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:39:36.995364125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:39:44.029780623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46954"
ts=2026-04-29T20:40:06.997333014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:40:14.03089492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42476"
ts=2026-04-29T20:40:16.369813412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:32938"
level=debug ts=2026-04-29T20:40:27.271849515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:40:36.999360247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:40:44.032664482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45918"
ts=2026-04-29T20:40:46.372637084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:49496"
ts=2026-04-29T20:41:00.234059907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43088"
ts=2026-04-29T20:41:07.002618836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T20:41:09.065337661Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:41:37.005677794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:41:46.378988901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:59412"
level=debug ts=2026-04-29T20:41:50.606309065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:42:07.008138066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:42:14.039269908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58996"
ts=2026-04-29T20:42:30.241385334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:56954"
ts=2026-04-29T20:42:37.010508935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:42:46.384663379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:41542"
level=debug ts=2026-04-29T20:42:51.00310454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:42:51.004724596Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:43:07.013014941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T20:43:33.261027853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:43:37.015710378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:44:07.01756974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:44:33.024832626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:44:37.019188461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:44:44.051124865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45842"
ts=2026-04-29T20:45:00.257161413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:36086"
ts=2026-04-29T20:45:07.021654245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:45:16.400241945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:43406"
level=debug ts=2026-04-29T20:45:18.026305288Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:45:37.023482109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:46:00.26208371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:33114"
level=debug ts=2026-04-29T20:46:01.406753822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:46:07.024582932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:46:30.264004447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48728"
ts=2026-04-29T20:46:37.026358091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:46:44.060132241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48710"
level=debug ts=2026-04-29T20:46:44.380350771Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:46:44.381919864Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:46:46.408539704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:47678"
ts=2026-04-29T20:47:07.027371205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:47:19.447483559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:47:37.030600395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:47:46.414310146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:37650"
ts=2026-04-29T20:48:07.033033579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:48:14.067141672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48424"
level=debug ts=2026-04-29T20:48:18.122259982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:48:20.626411905Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T20:48:20.62807288Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T20:48:20.629608977Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T20:48:20.629627942Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:48:30.274303303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:60336"
ts=2026-04-29T20:48:37.035829549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T20:49:01.958564591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:49:07.038336302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:49:37.041556702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
ts=2026-04-29T20:49:44.073559136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50142"
level=debug ts=2026-04-29T20:49:56.000193639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:50:07.043634298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:50:37.045476019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:50:42.92211098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:50:42.924014491Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:50:44.077900954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51488"
ts=2026-04-29T20:51:07.047858601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:51:25.986245273Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:51:30.292965553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:57296"
ts=2026-04-29T20:51:37.050772497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
ts=2026-04-29T20:51:44.081900122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39760"
ts=2026-04-29T20:52:07.053954563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e127a7f3 10.129.0.63:7946"
level=debug ts=2026-04-29T20:52:09.030902252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:52:14.083733198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52184"
ts=2026-04-29T20:52:30.298154244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:42254"
ts=2026-04-29T20:52:37.056599777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:52:46.446241311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:39722"
level=debug ts=2026-04-29T20:53:07.050146797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:53:07.058351637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:53:14.087840102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59444"
ts=2026-04-29T20:53:16.449373769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:38224"
ts=2026-04-29T20:53:37.060082745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:53:44.94964924Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:54:07.061845588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:54:14.091996768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48986"
ts=2026-04-29T20:54:37.064060556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:54:38.168819681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:54:38.17037879Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:54:44.093042887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42294"
ts=2026-04-29T20:55:07.065877556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:55:13.901565849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:55:37.067485026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:56:07.068500973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
ts=2026-04-29T20:56:14.100178347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50584"
level=debug ts=2026-04-29T20:56:14.974182766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"