level=info ts=2026-05-12T21:27:39.95071537Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-12T21:27:39.950801353Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-12T21:27:39.952715942Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-12T21:27:39.953069469Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-12T21:27:39.953145142Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.61 inf=eth0
level=debug ts=2026-05-12T21:27:39.954074744Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-12T21:27:39.954142221Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.61 inf=eth0
level=info ts=2026-05-12T21:27:39.954171655Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-f2155d35
ts=2026-05-12T21:27:39.954228934Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-12T21:27:39.954239575Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.61 advertisePort=7946
level=debug ts=2026-05-12T21:27:39.954253789Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.61 advertisePort=7946
level=info ts=2026-05-12T21:27:39.954381087Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-12T21:27:39.959005571Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T21:27:39.959028305Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:27:39.959039075Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=4.659928ms
level=error ts=2026-05-12T21:27:39.959049564Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-12T21:27:39.959062421Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-12T21:27:39.962029433Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:27:39.96204436Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-12T21:27:39.970003186Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-12T21:27:39.970006344Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-12T21:27:39.970027066Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-05-12T21:27:39.970022471Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-12T21:27:39.970050508Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-12T21:27:39.970136165Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-12T21:27:39.97021674Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-05-12T21:27:39.970225176Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-05-12T21:27:39.970228705Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-05-12T21:27:39.970231918Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-12T21:27:39.970304079Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-12T21:27:39.970331918Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-12T21:27:39.970346395Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-12T21:27:39.970444412Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-12T21:27:39.97045229Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-12T21:27:39.970461061Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-12T21:27:40.970949394Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-12T21:27:41.145253153Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-12T21:27:41.145320179Z caller=loki.go:615 msg="Loki started" startup_time=1.26102217s
ts=2026-05-12T21:27:41.252037477Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:27:41.252066117Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:27:44.224155485Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:27:44.224178545Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-12T21:27:46.145420547Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-12T21:27:46.145467132Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-12T21:27:50.088014675Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:27:50.088040481Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-12T21:28:00.953963912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:28:03.361735315Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.76:7946: dial tcp 10.128.0.76:7946: connect: connection refused"
level=warn ts=2026-05-12T21:28:03.361757013Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* failed to join 10.128.0.76:7946: dial tcp 10.128.0.76:7946: connect: connection refused\n\n"
level=debug ts=2026-05-12T21:28:20.303220978Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:28:25.701724348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:55488"
ts=2026-05-12T21:28:26.160371871Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.76:7946: dial tcp 10.128.0.76:7946: connect: connection refused"
ts=2026-05-12T21:28:26.161434094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.77:7946"
ts=2026-05-12T21:28:26.16375003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.74:7946"
ts=2026-05-12T21:28:26.16514052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.59:7946"
ts=2026-05-12T21:28:26.166221219Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.58:7946: dial tcp 10.130.0.58:7946: connect: connection refused"
ts=2026-05-12T21:28:26.166958699Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.60:7946: dial tcp 10.130.0.60:7946: connect: connection refused"
ts=2026-05-12T21:28:26.16703793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.61:7946"
ts=2026-05-12T21:28:26.167068648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55148"
level=info ts=2026-05-12T21:28:26.167850002Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=46.208780651s
ts=2026-05-12T21:28:27.775201345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T21:28:29.440980746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:37770"
ts=2026-05-12T21:28:31.364168037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:58592"
level=debug ts=2026-05-12T21:28:35.670044342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:28:49.798493139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:42316"
ts=2026-05-12T21:28:57.777885998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T21:29:03.424655308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:29:19.801899597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:52414"
ts=2026-05-12T21:29:20.85862901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:60578"
ts=2026-05-12T21:29:21.837233142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:60936"
ts=2026-05-12T21:29:27.781510029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:29:36.331516902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-12T21:29:36.333243213Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:29:50.860078392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:46864"
ts=2026-05-12T21:29:51.840067643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:36154"
ts=2026-05-12T21:29:57.784295806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T21:30:11.336559321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:30:27.786929866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T21:30:49.806704222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:57708"
ts=2026-05-12T21:30:50.865248055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:51330"
ts=2026-05-12T21:30:57.790156725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:31:20.206673136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:31:21.845954818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:56522"
ts=2026-05-12T21:31:27.793288917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T21:31:49.811775548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:52058"
ts=2026-05-12T21:31:50.869990937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:55972"
ts=2026-05-12T21:31:57.79610843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T21:32:19.815427815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:60778"
ts=2026-05-12T21:32:20.87152996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:40364"
ts=2026-05-12T21:32:27.799167584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:32:39.92946516Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:32:49.817850012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:58212"
ts=2026-05-12T21:32:57.801840994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T21:33:21.856916869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:57480"
ts=2026-05-12T21:33:27.804952325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:33:46.204278643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:33:50.879447539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:40886"
ts=2026-05-12T21:33:51.859907692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:53740"
ts=2026-05-12T21:33:57.807967596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T21:34:27.810446422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T21:34:53.849237755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-12T21:34:53.851578501Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:34:57.813249503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T21:35:27.815140816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T21:35:51.870859758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37214"
level=debug ts=2026-05-12T21:35:53.811209548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:35:57.817620315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T21:36:20.894400114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:57390"
ts=2026-05-12T21:36:21.873619038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:50818"
ts=2026-05-12T21:36:27.820380163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T21:36:49.831156989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:42090"
ts=2026-05-12T21:36:50.896320396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:40308"
ts=2026-05-12T21:36:51.876705207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34506"
level=debug ts=2026-05-12T21:36:53.404617037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:36:57.823917211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T21:37:21.879310597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:51520"
ts=2026-05-12T21:37:27.826658696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T21:37:31.59668951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T21:37:46.146382995Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-12T21:37:46.149790937Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:37:46.15184911Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:37:46.153815746Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:37:46.153843982Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T21:37:46.153857617Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-12T21:37:49.837725437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:34068"
ts=2026-05-12T21:37:57.829654321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T21:38:19.84093824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:52336"
ts=2026-05-12T21:38:20.903830051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:47338"
level=debug ts=2026-05-12T21:38:23.962001205Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:38:27.832178924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T21:38:49.845204467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:35912"
ts=2026-05-12T21:38:50.904953874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:46078"
ts=2026-05-12T21:38:51.886371054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:42300"
ts=2026-05-12T21:38:57.8339511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:39:11.260911793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:39:11.262674663Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:39:19.847988353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:47202"
ts=2026-05-12T21:39:20.906494002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:58162"
ts=2026-05-12T21:39:21.889179033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34806"
ts=2026-05-12T21:39:27.836173125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T21:39:49.85123475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:37400"
level=debug ts=2026-05-12T21:39:49.867115812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:39:57.837711191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T21:40:27.839164374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:40:38.865315904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:40:51.898245571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:45438"
ts=2026-05-12T21:40:57.840204148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T21:41:20.91666489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:57638"
ts=2026-05-12T21:41:21.900953423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:56090"
ts=2026-05-12T21:41:27.842494721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:41:35.710799447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:41:50.918431862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:33272"
ts=2026-05-12T21:41:51.903651528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:36872"
ts=2026-05-12T21:41:57.844549893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T21:42:27.846872357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:42:28.449454527Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:42:57.849965719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:43:10.997583722Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:43:10.999710919Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:43:19.866999741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:33502"
ts=2026-05-12T21:43:21.910537444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:33246"
ts=2026-05-12T21:43:27.851651827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T21:43:57.853692769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:44:10.922766464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:44:27.857107322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T21:44:50.935316489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:34694"
ts=2026-05-12T21:44:51.919215959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:32876"
ts=2026-05-12T21:44:57.860773186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T21:45:04.835122476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:45:27.86407047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T21:45:49.880834105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:56240"
ts=2026-05-12T21:45:57.866013701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:45:59.805889845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:46:27.868532215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T21:46:39.485410869Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:46:49.886925151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:45868"
ts=2026-05-12T21:46:57.8717937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T21:47:19.890285936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:57418"
ts=2026-05-12T21:47:20.950820328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:48542"
level=debug ts=2026-05-12T21:47:23.395100919Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:47:23.397195071Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:47:27.873707505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=error ts=2026-05-12T21:47:46.159535483Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:47:46.161930537Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:47:46.163841202Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:47:46.163861963Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:47:49.89390677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:60426"
ts=2026-05-12T21:47:50.952509517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:50872"
ts=2026-05-12T21:47:57.874965336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:48:12.612197947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:48:20.954249174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:57518"
ts=2026-05-12T21:48:27.877338469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T21:48:50.956305511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:43588"
ts=2026-05-12T21:48:57.880649627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:49:04.468822292Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:49:27.884019117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T21:49:51.940901866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:47464"
level=debug ts=2026-05-12T21:49:55.267125389Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:49:57.886862764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T21:50:19.906993347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:47392"
ts=2026-05-12T21:50:27.890152778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T21:50:33.088506253Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:50:51.945448549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:40548"
ts=2026-05-12T21:50:57.893671784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T21:51:27.897002506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:51:31.482118486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:51:31.48411303Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:51:49.916333707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:46236"
ts=2026-05-12T21:51:57.899095029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:52:06.76757475Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:52:21.953138775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:56486"
ts=2026-05-12T21:52:27.901228233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T21:52:51.956523642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:42480"
level=debug ts=2026-05-12T21:52:52.844057541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:52:57.903789694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T21:53:19.925651161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:34858"
ts=2026-05-12T21:53:21.95996567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:46854"
ts=2026-05-12T21:53:27.905803117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T21:53:49.928335036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:55372"
level=debug ts=2026-05-12T21:53:50.38366386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:53:57.909461824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T21:54:21.967011457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:58898"
ts=2026-05-12T21:54:27.912353555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:54:41.445117341Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:54:49.936528288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:54078"
ts=2026-05-12T21:54:57.916086664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T21:55:20.996295979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:55474"
level=debug ts=2026-05-12T21:55:21.241807047Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:55:21.24393335Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:55:21.970716557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:42994"
ts=2026-05-12T21:55:27.918857527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T21:55:49.940892376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:40120"
ts=2026-05-12T21:55:50.998014771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:37768"
ts=2026-05-12T21:55:57.92215509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:56:14.150563161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:56:21.976538756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:53276"
ts=2026-05-12T21:56:27.926051784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T21:56:49.945588634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:55410"
ts=2026-05-12T21:56:51.979849011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52792"
ts=2026-05-12T21:56:57.929192331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T21:57:15.721055006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:57:27.931713445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=error ts=2026-05-12T21:57:46.161960794Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:57:46.164831129Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:57:46.168195183Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:57:46.168233523Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:57:51.986044125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:33124"
ts=2026-05-12T21:57:57.933688805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T21:58:15.585527827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:58:27.937032047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T21:58:57.943693771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:59:04.226386692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:59:21.023211835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:37390"
ts=2026-05-12T21:59:21.992634203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:51472"
ts=2026-05-12T21:59:27.945810342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:59:41.997792828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:59:42.000074068Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:59:49.972099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:60598"
ts=2026-05-12T21:59:57.947695888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T22:00:16.574069184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:00:19.975382681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:37156"
ts=2026-05-12T22:00:21.027558671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:57812"
ts=2026-05-12T22:00:27.950959443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:00:51.028950597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:37706"
ts=2026-05-12T22:00:57.953599731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:00:58.735689593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:01:22.003510616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:45718"
ts=2026-05-12T22:01:27.956020336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:01:39.467208352Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:01:49.982809372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:59538"
ts=2026-05-12T22:01:52.006235105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:44332"
ts=2026-05-12T22:01:57.959699871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T22:02:15.819393714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:02:19.98839541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:56742"
ts=2026-05-12T22:02:27.962400614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:02:51.044856427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:35462"
ts=2026-05-12T22:02:52.0109016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:45402"
ts=2026-05-12T22:02:57.965317202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:03:07.300684138Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:03:07.302713862Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:03:21.046795945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:50156"
ts=2026-05-12T22:03:22.0149818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:55686"
ts=2026-05-12T22:03:27.968235522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:03:50.954474031Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:03:51.04882847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:60446"
ts=2026-05-12T22:03:57.970992423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:04:22.02200266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:43340"
ts=2026-05-12T22:04:27.973313615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:04:33.837296806Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:04:57.974359416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:05:21.057931614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:43730"
level=debug ts=2026-05-12T22:05:22.616602337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:05:27.976214644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:05:51.059121146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:34552"
ts=2026-05-12T22:05:57.978243838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:06:13.47019481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:06:22.028686222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:49018"
ts=2026-05-12T22:06:27.980300288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:06:51.063531469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:38198"
ts=2026-05-12T22:06:52.032527791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:40056"
ts=2026-05-12T22:06:57.982659884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:07:07.022741384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:07:07.024496935Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:07:21.064562637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:58932"
ts=2026-05-12T22:07:27.986184381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T22:07:43.332347477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:07:46.1580271Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:07:46.16002147Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:07:46.161840551Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:07:46.161862935Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:07:57.989373746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:08:22.04072864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:40932"
ts=2026-05-12T22:08:27.991776701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:08:44.641647816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:08:52.043621249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:46150"
ts=2026-05-12T22:08:57.993850632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:09:20.03017682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:50266"
ts=2026-05-12T22:09:21.075956958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:57482"
ts=2026-05-12T22:09:22.047029504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:40004"
ts=2026-05-12T22:09:27.996236597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:09:44.633437492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:09:52.049936023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:51072"
ts=2026-05-12T22:09:57.997348486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:10:20.035721197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:34276"
ts=2026-05-12T22:10:21.0805535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:43130"
ts=2026-05-12T22:10:22.053609693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:42990"
ts=2026-05-12T22:10:27.999352287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:10:44.371014197Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:10:52.056318827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:41038"
ts=2026-05-12T22:10:58.001821185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:11:28.003893691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:11:39.30642893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:11:39.308128643Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:11:50.046363787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:42748"
ts=2026-05-12T22:11:51.08712002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:35424"
ts=2026-05-12T22:11:58.004869136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:12:20.049833553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:40874"
ts=2026-05-12T22:12:21.088847566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:58140"
ts=2026-05-12T22:12:22.066097923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:49132"
ts=2026-05-12T22:12:28.006864816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T22:12:30.64685105Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:12:50.052556425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:50196"
ts=2026-05-12T22:12:58.009027702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:13:12.879403352Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:13:20.056046141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:56394"
ts=2026-05-12T22:13:28.011629714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:13:50.850455692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:13:58.014390397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:14:20.060291046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:34130"
ts=2026-05-12T22:14:28.016365674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:14:42.819759562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:14:50.063334135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:53234"
ts=2026-05-12T22:14:58.018871902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:15:22.082234081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:59740"
ts=2026-05-12T22:15:28.022068522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:15:28.944144091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:15:28.945728127Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:15:58.025561735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:16:21.113555991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:37682"
ts=2026-05-12T22:16:22.087903041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:53460"
level=debug ts=2026-05-12T22:16:22.401727195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:16:28.028712814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:16:51.115147877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:47222"
ts=2026-05-12T22:16:58.032512757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:17:18.938347445Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:17:28.034638599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=error ts=2026-05-12T22:17:46.158613037Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:17:46.160380627Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:17:46.162064303Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:17:46.162085008Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:17:51.119642938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:44578"
ts=2026-05-12T22:17:58.036416723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:17:58.185355871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:18:22.097354385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:38234"
ts=2026-05-12T22:18:28.039042869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:18:36.412187358Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:18:50.082236502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:38968"
ts=2026-05-12T22:18:58.042352335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T22:19:21.894407187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:19:21.896213673Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:19:28.045912307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:19:51.131943059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:35324"
ts=2026-05-12T22:19:58.048152156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:20:21.133137911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:60566"
level=debug ts=2026-05-12T22:20:22.006447681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:20:28.050916836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:20:51.134373104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:56148"
level=debug ts=2026-05-12T22:20:57.973216643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:20:58.052715578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:21:28.053708506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:21:35.413437885Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:21:50.09296867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:60654"
ts=2026-05-12T22:21:52.113968459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52848"
ts=2026-05-12T22:21:58.055705533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:22:19.149950494Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:22:21.142128056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:53660"
ts=2026-05-12T22:22:22.117248265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:49304"
ts=2026-05-12T22:22:28.058632852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:22:58.060724594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:23:06.549639609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:23:06.551107785Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:23:21.146583259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:56118"
ts=2026-05-12T22:23:28.063335288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:23:58.065168528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:23:58.773646894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:24:21.151584259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:55492"
ts=2026-05-12T22:24:22.125296645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:46036"
ts=2026-05-12T22:24:28.068005844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T22:24:43.783087183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:24:50.112306945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:50642"
ts=2026-05-12T22:24:51.152772431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:58332"
ts=2026-05-12T22:24:58.070140539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:25:22.057772008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:25:28.072556392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:25:51.157476651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:59784"
ts=2026-05-12T22:25:58.075157603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:26:04.531277882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:26:21.158657738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:47178"
ts=2026-05-12T22:26:28.076014181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:26:51.159815161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:36362"
level=debug ts=2026-05-12T22:26:54.999121692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:26:55.001024139Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:26:58.078675075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:27:20.1247454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:42602"
ts=2026-05-12T22:27:21.161237496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:48628"
ts=2026-05-12T22:27:22.141789408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:35530"
ts=2026-05-12T22:27:28.080573073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=error ts=2026-05-12T22:27:46.159570823Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:27:46.161690288Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:27:46.163586252Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:27:46.163610575Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:27:50.128013071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:56058"
level=debug ts=2026-05-12T22:27:56.109998978Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:27:58.081812119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:28:20.131220033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:35376"
ts=2026-05-12T22:28:21.165468426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:36050"
ts=2026-05-12T22:28:22.145818049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:36578"
ts=2026-05-12T22:28:28.083309608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:28:45.98704886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:28:50.13447454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:38616"
ts=2026-05-12T22:28:58.085809905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:29:20.137801881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:46710"
ts=2026-05-12T22:29:22.15094355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:54932"
ts=2026-05-12T22:29:28.088525646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T22:29:34.870690344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:29:58.091307124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:30:22.156593775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34174"
level=debug ts=2026-05-12T22:30:23.958844935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:30:28.094240089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:30:50.146173081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:37502"
ts=2026-05-12T22:30:51.182417147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:34578"
ts=2026-05-12T22:30:58.096765697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:31:14.423938541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:31:14.425465229Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:31:22.162692408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:56254"
ts=2026-05-12T22:31:28.098053717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:31:50.151790368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:51680"
ts=2026-05-12T22:31:52.165712621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:54372"
ts=2026-05-12T22:31:58.10038702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T22:32:14.768828853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:32:21.19120778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:58252"
ts=2026-05-12T22:32:28.103218042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:32:51.192222916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:48698"
level=debug ts=2026-05-12T22:32:55.133960007Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:32:58.105403056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:33:28.107538211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:33:54.335418634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:33:58.110695073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:34:28.112938072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:34:29.269588221Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:34:58.115192761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T22:35:19.453188302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:35:19.454800323Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:35:20.173324747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:39382"
ts=2026-05-12T22:35:21.208403858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:43388"
ts=2026-05-12T22:35:28.116816496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:35:58.118342162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:36:04.637554731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:36:20.178592126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:53670"
ts=2026-05-12T22:36:22.189093511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:56676"
ts=2026-05-12T22:36:28.120367594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T22:36:46.07410976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:36:58.122750508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:37:20.183539753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:48406"
ts=2026-05-12T22:37:21.219174462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:40766"
ts=2026-05-12T22:37:28.123691731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:37:39.797975102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:37:46.15813309Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:37:46.159945036Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:37:46.161561509Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:37:46.161580262Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:37:51.22077004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:47516"
ts=2026-05-12T22:37:58.125321761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:38:19.096312511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:38:20.187804338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:41680"
ts=2026-05-12T22:38:21.222880972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:43772"
ts=2026-05-12T22:38:22.199752786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:54542"
ts=2026-05-12T22:38:28.126129229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:38:58.129151313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T22:39:20.276701793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:39:20.278816937Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:39:28.131629311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:39:51.231588886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:55176"
ts=2026-05-12T22:39:58.133191846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:39:58.239782131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:40:22.210464305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:55070"
ts=2026-05-12T22:40:28.135750746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:40:58.109740611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:40:58.138996032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:41:21.239750013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:57932"
ts=2026-05-12T22:41:22.214342093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:60302"
ts=2026-05-12T22:41:28.141357591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:41:47.942082789Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:41:50.203058467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:49964"
ts=2026-05-12T22:41:58.143362578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:42:20.205887931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:54512"
ts=2026-05-12T22:42:22.219234191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:48352"
ts=2026-05-12T22:42:28.146020554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:42:40.74678507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:42:50.209211361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:37974"
ts=2026-05-12T22:42:58.14784157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T22:43:21.17115446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:43:21.172806091Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:43:21.251275193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:58508"
ts=2026-05-12T22:43:28.151311046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:43:58.154563266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:44:13.510540784Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:44:20.218510571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:54640"
ts=2026-05-12T22:44:28.158013864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:44:50.221626069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:54830"
ts=2026-05-12T22:44:58.160034685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:45:10.230621052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:45:20.225094066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:47408"
ts=2026-05-12T22:45:22.234454612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:41350"
ts=2026-05-12T22:45:28.162519027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T22:45:47.992384729Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:45:58.165910491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:46:20.231522811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:53622"
ts=2026-05-12T22:46:22.239306755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37198"
ts=2026-05-12T22:46:28.169202752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:46:36.368715653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:46:52.242016438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:59318"
ts=2026-05-12T22:46:58.172709127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:47:28.17450314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:47:34.000945521Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:47:34.002423369Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:47:46.157196755Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:47:46.159110555Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:47:46.160943758Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:47:46.160966037Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:47:50.242859146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:55088"
ts=2026-05-12T22:47:51.279993434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:33216"
ts=2026-05-12T22:47:58.176427565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T22:48:08.930092535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:48:21.280954108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:55518"
ts=2026-05-12T22:48:28.178804365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:48:52.249733843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:45076"
ts=2026-05-12T22:48:58.179800822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:49:08.965395848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:49:28.181192853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:49:46.761440409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:49:50.254128081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:55978"
ts=2026-05-12T22:49:52.256031193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52476"
ts=2026-05-12T22:49:58.184033838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:50:28.186953945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T22:50:37.561788157Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:50:51.295960451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:36744"
ts=2026-05-12T22:50:58.189290707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:51:22.264761532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:56846"
ts=2026-05-12T22:51:28.191596671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:51:32.836987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:51:32.838987595Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:51:50.261768315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:49088"
ts=2026-05-12T22:51:58.193611108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:52:19.087784316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:52:20.264665205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:33340"
ts=2026-05-12T22:52:22.27109399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:33380"
ts=2026-05-12T22:52:28.195824825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:52:58.199137029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:53:01.931527269Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:53:20.270788454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:40232"
ts=2026-05-12T22:53:21.309649489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:58392"
ts=2026-05-12T22:53:28.202199611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:53:40.71742309Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:53:50.274433165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:36360"
ts=2026-05-12T22:53:58.205759557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:54:21.314348499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:45390"
level=debug ts=2026-05-12T22:54:23.39348067Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:54:28.209185165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:54:58.211933002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:55:01.460936287Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:55:01.462882037Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:55:28.21518866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:55:37.606541809Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:55:50.286032608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:51330"
ts=2026-05-12T22:55:58.217224068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:56:27.737631305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:56:28.219238653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:56:51.32837892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:50544"
ts=2026-05-12T22:56:58.222623665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T22:57:02.189755588Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:57:20.291613848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:49986"
ts=2026-05-12T22:57:22.289702784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:32840"
ts=2026-05-12T22:57:28.22553215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=error ts=2026-05-12T22:57:46.155957712Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:57:46.15760165Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:57:46.15929272Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:57:46.159312991Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T22:57:48.290704535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:57:58.228087109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:58:28.228890512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:58:40.963892964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:58:40.965653201Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:58:58.231294669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T22:59:21.342689648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:39238"
ts=2026-05-12T22:59:28.234475677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:59:32.776561414Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:59:50.304701973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:37220"
ts=2026-05-12T22:59:51.343909529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:50376"
ts=2026-05-12T22:59:58.236835827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T23:00:08.585160068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T23:00:22.303670043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37280"
ts=2026-05-12T23:00:28.240091851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T23:00:44.221203502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T23:00:52.30641776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:40732"
ts=2026-05-12T23:00:58.243192142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T23:01:20.829814837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T23:01:21.351348124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:35002"
ts=2026-05-12T23:01:22.309202996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:45144"
ts=2026-05-12T23:01:28.246312873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T23:01:55.273871123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T23:01:55.275605267Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T23:01:58.24912812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T23:02:20.315616458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:33068"
ts=2026-05-12T23:02:21.355423407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:38670"
ts=2026-05-12T23:02:28.250897205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T23:02:40.445456911Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T23:02:51.357164853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:37134"
ts=2026-05-12T23:02:52.317354111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:60686"
ts=2026-05-12T23:02:58.252663999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T23:03:21.358232112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:60966"
ts=2026-05-12T23:03:28.255238259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T23:03:34.027433485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T23:03:52.321334082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:51500"
ts=2026-05-12T23:03:58.258092568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T23:04:20.327876424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:54326"
ts=2026-05-12T23:04:28.260671073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T23:04:30.764355003Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T23:04:58.263010728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T23:05:17.19603574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T23:05:20.334223083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:49050"
ts=2026-05-12T23:05:22.329538003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52186"
ts=2026-05-12T23:05:28.265834604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T23:05:52.332427012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:55900"
ts=2026-05-12T23:05:58.269062166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T23:05:59.828201867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T23:05:59.83008889Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T23:06:21.376183482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:60588"
ts=2026-05-12T23:06:28.272386237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
ts=2026-05-12T23:06:58.275441243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T23:07:00.951978662Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T23:07:22.339029649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:40860"
ts=2026-05-12T23:07:28.278683259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T23:07:42.420658542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T23:07:46.156682441Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T23:07:46.15829389Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T23:07:46.159823947Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T23:07:46.159847791Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T23:07:52.341760557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34354"
ts=2026-05-12T23:07:58.281908683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T23:08:21.432077276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T23:08:28.284325163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T23:08:52.346646175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:35802"
level=debug ts=2026-05-12T23:08:56.455500797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T23:08:58.285342732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T23:09:21.394857674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:47716"
ts=2026-05-12T23:09:22.349902668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:35536"
ts=2026-05-12T23:09:28.287238843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T23:09:40.612939903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T23:09:40.614588058Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T23:09:58.288956973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T23:10:28.291126996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T23:10:28.716016511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T23:10:52.357192673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:35148"
ts=2026-05-12T23:10:58.294072493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T23:11:05.465890741Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T23:11:22.36065607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:54564"
ts=2026-05-12T23:11:28.296615462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T23:11:50.749893873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T23:11:58.299557255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T23:12:20.368094517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:40776"
ts=2026-05-12T23:12:21.409521456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:40190"
ts=2026-05-12T23:12:28.301480223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T23:12:31.137572606Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T23:12:50.371419656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:33174"
ts=2026-05-12T23:12:51.410751771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:41124"
ts=2026-05-12T23:12:58.303837305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-529f4c4f 10.128.0.77:7946"
level=debug ts=2026-05-12T23:13:18.55587844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T23:13:18.557805999Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T23:13:20.374224615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:55960"
ts=2026-05-12T23:13:22.371043035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39820"