level=info ts=2026-05-16T11:12:03.192169768Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-16T11:12:03.192193456Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-16T11:12:03.193833029Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-16T11:12:03.194248385Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-16T11:12:03.19432909Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.84 inf=eth0
level=debug ts=2026-05-16T11:12:03.194724043Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-16T11:12:03.194804619Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.84 inf=eth0
level=info ts=2026-05-16T11:12:03.194902743Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-5fe08e1d
ts=2026-05-16T11:12:03.194986134Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-16T11:12:03.194996912Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.84 advertisePort=7946
level=debug ts=2026-05-16T11:12:03.195012416Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.84 advertisePort=7946
level=info ts=2026-05-16T11:12:03.195125543Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-16T11:12:03.201393871Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-16T11:12:03.201413373Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-16T11:12:03.201426146Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.302878ms
level=error ts=2026-05-16T11:12:03.201438313Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-16T11:12:03.201453111Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-16T11:12:03.20425346Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-16T11:12:03.204268953Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-16T11:12:03.208747908Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-16T11:12:03.208786923Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-16T11:12:03.208785477Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-05-16T11:12:03.208809124Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-05-16T11:12:03.208811264Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-16T11:12:03.208817546Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-16T11:12:03.208850706Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-16T11:12:03.209073021Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-05-16T11:12:03.209081052Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-16T11:12:03.209086156Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-16T11:12:03.209155334Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-16T11:12:03.209167168Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-16T11:12:03.2092387Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-16T11:12:03.209452361Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-16T11:12:03.209479166Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-16T11:12:03.209506514Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-16T11:12:04.209910114Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-16T11:12:04.406445489Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-16T11:12:04.406494077Z caller=loki.go:615 msg="Loki started" startup_time=1.277552259s
ts=2026-05-16T11:12:04.748988924Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-16T11:12:04.749012551Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-16T11:12:07.932746053Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-16T11:12:07.932801739Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-16T11:12:09.407415957Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-16T11:12:09.407467906Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-16T11:12:13.964555745Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-16T11:12:13.964579376Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-16T11:12:17.391309547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-16T11:12:24.494332097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:58530"
ts=2026-05-16T11:12:24.494377607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.84:7946"
level=info ts=2026-05-16T11:12:24.494647843Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=21.293187471s
ts=2026-05-16T11:12:25.743112961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:35784"
ts=2026-05-16T11:12:31.067604276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:56048"
level=debug ts=2026-05-16T11:12:31.472259429Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-16T11:12:34.204192955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:39858"
ts=2026-05-16T11:12:55.611117767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=debug ts=2026-05-16T11:12:57.44801163Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-16T11:13:13.516185376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:43686"
level=debug ts=2026-05-16T11:13:16.567722915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-16T11:13:25.613479492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:13:31.219802946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:41528"
level=debug ts=2026-05-16T11:13:46.920334841Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-16T11:13:46.921898699Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T11:13:55.616247525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:13:57.668303101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:60636"
ts=2026-05-16T11:14:13.519565657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:51192"
level=debug ts=2026-05-16T11:14:22.104219084Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-16T11:14:25.617963983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:14:27.670954417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:50460"
ts=2026-05-16T11:14:43.521426384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:54442"
ts=2026-05-16T11:14:55.620118726Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:15:01.228379701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:52476"
ts=2026-05-16T11:15:25.622538679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
level=debug ts=2026-05-16T11:15:26.150807347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-16T11:15:31.230718405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56584"
ts=2026-05-16T11:15:43.525556646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:52752"
ts=2026-05-16T11:15:55.625719476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:16:13.527203178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:41156"
ts=2026-05-16T11:16:25.628091315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:16:27.684119376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:53348"
level=debug ts=2026-05-16T11:16:36.220195816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-16T11:16:55.630586459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:17:01.238922554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:39454"
ts=2026-05-16T11:17:13.531091728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:38698"
ts=2026-05-16T11:17:25.633358145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:17:31.240694664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:39406"
level=debug ts=2026-05-16T11:17:38.463905908Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-16T11:17:55.635473491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:17:57.694127301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:52374"
ts=2026-05-16T11:18:13.535026323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:35890"
ts=2026-05-16T11:18:25.638049762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=debug ts=2026-05-16T11:18:37.012360493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-16T11:18:37.014375406Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T11:18:43.537212129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:60710"
ts=2026-05-16T11:18:55.640086217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:19:13.539290918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:46832"
ts=2026-05-16T11:19:25.641925969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=debug ts=2026-05-16T11:19:34.25405548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:19:43.541889556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:47728"
ts=2026-05-16T11:19:55.645044361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:20:25.648508296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
level=debug ts=2026-05-16T11:20:33.04992586Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:20:55.651898667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:20:57.713051333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:52616"
ts=2026-05-16T11:21:01.255136391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:58100"
level=debug ts=2026-05-16T11:21:19.428345371Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:21:25.654482673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:21:31.257226666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:34878"
ts=2026-05-16T11:21:55.656552279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:21:57.719243251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:46878"
level=info ts=2026-05-16T11:22:09.408415851Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-16T11:22:09.412982564Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T11:22:09.414838693Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T11:22:09.416641072Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T11:22:09.416665865Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-16T11:22:09.416677818Z caller=tables_manager.go:148 msg="compactor started"
level=debug ts=2026-05-16T11:22:21.24946666Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:22:25.659088355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=debug ts=2026-05-16T11:22:55.654481573Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T11:22:55.656252577Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T11:22:55.66100176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:23:25.663511011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
level=debug ts=2026-05-16T11:23:42.149320006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:23:55.66627552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:24:01.271329974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:51436"
ts=2026-05-16T11:24:25.668903026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:24:31.27320581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:51852"
level=debug ts=2026-05-16T11:24:34.714132381Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:24:55.671275946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:24:57.737496951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:47154"
level=debug ts=2026-05-16T11:25:10.186816913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:25:25.672826037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:25:27.740446754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:38930"
ts=2026-05-16T11:25:43.56830378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:35308"
level=debug ts=2026-05-16T11:25:52.52983014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:25:55.675463214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:25:57.743942774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:48764"
ts=2026-05-16T11:26:13.570724383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:58836"
ts=2026-05-16T11:26:25.678316161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
level=debug ts=2026-05-16T11:26:29.435824741Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T11:26:29.437583577Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T11:26:55.681159298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=debug ts=2026-05-16T11:27:09.035340532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:27:25.683439134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=debug ts=2026-05-16T11:27:48.664682311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:27:55.685455316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:28:25.687928221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=debug ts=2026-05-16T11:28:32.859702541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:28:55.690552395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
level=debug ts=2026-05-16T11:29:09.158590524Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:29:25.693496515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:29:27.763857661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:38634"
ts=2026-05-16T11:29:43.584488844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:60206"
ts=2026-05-16T11:29:55.695314731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:29:57.766489345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:51296"
level=debug ts=2026-05-16T11:30:01.015581718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T11:30:01.017089057Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T11:30:13.587005782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:37810"
ts=2026-05-16T11:30:25.697208215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:30:27.769671746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:40826"
ts=2026-05-16T11:30:31.296274706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:47452"
ts=2026-05-16T11:30:55.699319288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=debug ts=2026-05-16T11:30:59.799325788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:31:25.701882192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:31:31.299709376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42480"
level=debug ts=2026-05-16T11:31:37.450167737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:31:55.703441044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:31:57.779000494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:49618"
level=error ts=2026-05-16T11:32:09.423167131Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T11:32:09.42523858Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T11:32:09.42725069Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T11:32:09.42727327Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T11:32:25.705589626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:32:27.782433137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:43172"
ts=2026-05-16T11:32:31.303287617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:35820"
level=debug ts=2026-05-16T11:32:38.271706736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:32:55.708125603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=debug ts=2026-05-16T11:33:17.359375703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:33:25.710967838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:33:27.788938321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:58172"
ts=2026-05-16T11:33:31.307496224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:57906"
ts=2026-05-16T11:33:55.713892287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:34:01.309992342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:48588"
ts=2026-05-16T11:34:13.606306377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:60254"
level=debug ts=2026-05-16T11:34:19.332535872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T11:34:19.334308103Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T11:34:25.716815057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:34:55.719947747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:34:57.798222642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:38112"
level=debug ts=2026-05-16T11:35:10.411372832Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:35:25.72267815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:35:27.801503412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:50336"
ts=2026-05-16T11:35:43.610669631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:38506"
ts=2026-05-16T11:35:55.725321878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=debug ts=2026-05-16T11:36:10.593620213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:36:13.613147476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:46452"
ts=2026-05-16T11:36:25.728150944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:36:43.61479584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:41680"
ts=2026-05-16T11:36:55.730755555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:36:57.810394183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:41248"
ts=2026-05-16T11:37:01.324154348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42570"
level=debug ts=2026-05-16T11:37:08.09230777Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:37:13.616977478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:49260"
ts=2026-05-16T11:37:25.73374161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:37:55.736662335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=debug ts=2026-05-16T11:38:00.214421956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:38:25.738524292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:38:27.818790585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:52050"
ts=2026-05-16T11:38:31.330970242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:53660"
level=debug ts=2026-05-16T11:38:34.883325829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T11:38:34.885057052Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T11:38:43.622352443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:46418"
ts=2026-05-16T11:38:55.740403323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:39:01.333224466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:34152"
ts=2026-05-16T11:39:25.74390043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
level=debug ts=2026-05-16T11:39:28.026252771Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:39:55.746866663Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:39:57.827874568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:59642"
level=debug ts=2026-05-16T11:40:17.021815085Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:40:25.749540076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:40:43.63184509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:37654"
ts=2026-05-16T11:40:55.751346456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=debug ts=2026-05-16T11:40:57.027105851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:41:13.634317371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:48554"
ts=2026-05-16T11:41:25.753436935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
level=debug ts=2026-05-16T11:41:55.42999877Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:41:55.756655606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:41:57.840404154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:58016"
level=error ts=2026-05-16T11:42:09.420218081Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T11:42:09.422333216Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T11:42:09.424137291Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T11:42:09.424157039Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T11:42:13.640081141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:52970"
ts=2026-05-16T11:42:25.759050972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:42:31.345369178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42284"
level=debug ts=2026-05-16T11:42:45.347646988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T11:42:45.349385235Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T11:42:55.761273235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:42:57.846658718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:44408"
ts=2026-05-16T11:43:01.34779543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46006"
level=debug ts=2026-05-16T11:43:21.604621604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:43:25.763857757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:43:27.84950753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:35720"
ts=2026-05-16T11:43:55.765783694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:44:13.648491965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:35744"
level=debug ts=2026-05-16T11:44:18.463196399Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:44:25.767587408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:44:55.770229629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:45:13.653452013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:42416"
level=debug ts=2026-05-16T11:45:16.90748239Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:45:25.773118717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:45:31.359410804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46162"
ts=2026-05-16T11:45:43.655261217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:59292"
ts=2026-05-16T11:45:55.776048778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
level=debug ts=2026-05-16T11:46:04.981355713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:46:25.778503074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=debug ts=2026-05-16T11:46:40.093528923Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T11:46:40.094990875Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T11:46:55.78033905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:46:57.871754674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:48006"
level=debug ts=2026-05-16T11:47:24.109621472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:47:25.782750197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:47:55.785105105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=debug ts=2026-05-16T11:48:21.432949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:48:25.787506211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:48:31.369970004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:44618"
ts=2026-05-16T11:48:43.671229496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:44572"
ts=2026-05-16T11:48:55.789547864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=debug ts=2026-05-16T11:48:57.385560008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:49:25.791602946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:49:31.375581064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46292"
ts=2026-05-16T11:49:43.676204764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:33460"
level=debug ts=2026-05-16T11:49:45.484232783Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:49:55.793479196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:49:57.8912659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:51130"
level=debug ts=2026-05-16T11:50:23.006491962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T11:50:23.008129677Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T11:50:25.796074226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:50:31.378690455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56594"
ts=2026-05-16T11:50:55.798908401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=debug ts=2026-05-16T11:50:59.560732235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:51:01.381154706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:39490"
ts=2026-05-16T11:51:25.801384808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:51:43.685399643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:59422"
level=debug ts=2026-05-16T11:51:45.319279192Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:51:55.803593077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:52:01.386596945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46024"
level=error ts=2026-05-16T11:52:09.420994706Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T11:52:09.42306576Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T11:52:09.424892819Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T11:52:09.424913048Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T11:52:13.687220614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:37720"
ts=2026-05-16T11:52:25.80595407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:52:27.907090857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:58956"
ts=2026-05-16T11:52:31.388921465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42552"
level=debug ts=2026-05-16T11:52:42.677523473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:52:43.689489207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:40640"
ts=2026-05-16T11:52:55.809066688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:52:57.910668229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:36324"
ts=2026-05-16T11:53:25.810925387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:53:27.91360967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:43866"
level=debug ts=2026-05-16T11:53:42.093648682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:53:55.813472927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:53:57.917163066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:53118"
level=debug ts=2026-05-16T11:54:17.561623527Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T11:54:17.563529159Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T11:54:25.817108711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:54:27.920718116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:34998"
level=debug ts=2026-05-16T11:54:55.778421023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:54:55.819372429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:55:01.400215499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:39420"
ts=2026-05-16T11:55:25.821473246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:55:43.704846905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:41228"
level=debug ts=2026-05-16T11:55:55.469917331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:55:55.825143838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:56:01.404503457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:43616"
ts=2026-05-16T11:56:25.828061998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:56:31.406359118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56846"
level=debug ts=2026-05-16T11:56:34.66682633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:56:55.83051167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:57:01.408493887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:45256"
level=debug ts=2026-05-16T11:57:10.891349148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:57:13.711231773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:59532"
ts=2026-05-16T11:57:25.832998328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:57:43.713841125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:50992"
ts=2026-05-16T11:57:55.835407611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=debug ts=2026-05-16T11:57:57.499394719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T11:57:57.500847297Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T11:58:13.716291732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:44394"
ts=2026-05-16T11:58:25.837839927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T11:58:27.944903851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:47952"
level=debug ts=2026-05-16T11:58:49.644324373Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:58:55.841028057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T11:59:25.843846822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T11:59:27.950972874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:34454"
level=debug ts=2026-05-16T11:59:51.112909386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T11:59:55.846293521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:00:25.848695119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=debug ts=2026-05-16T12:00:26.520902546Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:00:27.95732187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:52532"
ts=2026-05-16T12:00:31.426386353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:40652"
ts=2026-05-16T12:00:55.851066938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=debug ts=2026-05-16T12:01:09.628064875Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:01:25.853516623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:01:31.430924369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:59392"
level=debug ts=2026-05-16T12:01:51.509832187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T12:01:51.511275577Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T12:01:55.85596353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=error ts=2026-05-16T12:02:09.421037166Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T12:02:09.423281478Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T12:02:09.425209145Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T12:02:09.425236232Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T12:02:25.857633416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:02:27.9695338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56238"
level=debug ts=2026-05-16T12:02:29.130152765Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:02:55.860124964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:02:57.972280734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:57300"
level=debug ts=2026-05-16T12:03:06.231569204Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:03:25.862972377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:03:55.866036712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:03:57.979489227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:41280"
level=debug ts=2026-05-16T12:04:06.388923996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:04:13.740421309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:49886"
ts=2026-05-16T12:04:25.86901211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:04:55.871126291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:05:01.446215985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56716"
level=debug ts=2026-05-16T12:05:01.705270075Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:05:25.873546178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:05:43.745964116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:44588"
level=debug ts=2026-05-16T12:05:52.688726219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T12:05:52.69061127Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T12:05:55.875337268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:05:57.993168343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:60048"
ts=2026-05-16T12:06:25.877999163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:06:31.453030191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:41188"
level=debug ts=2026-05-16T12:06:42.496901238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:06:43.749795787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:39748"
ts=2026-05-16T12:06:55.880367342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:07:01.455246934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:36378"
ts=2026-05-16T12:07:25.882794551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=debug ts=2026-05-16T12:07:28.080061673Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:07:55.88506398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
level=debug ts=2026-05-16T12:08:15.766699053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:08:25.887398949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:08:31.460458902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:49316"
ts=2026-05-16T12:08:43.756511634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:42148"
level=debug ts=2026-05-16T12:08:50.941738153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:08:55.889885211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:09:25.892456345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:09:28.013819703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:37714"
ts=2026-05-16T12:09:31.465089992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:38046"
level=debug ts=2026-05-16T12:09:51.688083986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T12:09:51.690074776Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T12:09:55.895012891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:10:25.89724121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:10:28.020556093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:43540"
ts=2026-05-16T12:10:31.469172778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:59552"
level=debug ts=2026-05-16T12:10:41.038236565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:10:55.900172151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:11:13.768136755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:52478"
ts=2026-05-16T12:11:25.902477943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:11:31.474296014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:45918"
level=debug ts=2026-05-16T12:11:41.964530323Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:11:43.769711755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:46076"
ts=2026-05-16T12:11:55.904511008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:11:58.030265151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:59744"
level=error ts=2026-05-16T12:12:09.420612826Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T12:12:09.422539594Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T12:12:09.424180468Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T12:12:09.424204362Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T12:12:25.907502602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
level=debug ts=2026-05-16T12:12:35.051936078Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:12:43.773234488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:33934"
ts=2026-05-16T12:12:55.910519981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:13:13.775620009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:39278"
ts=2026-05-16T12:13:25.912230881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=debug ts=2026-05-16T12:13:27.78093183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:13:28.040326312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:50544"
ts=2026-05-16T12:13:55.914715749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
level=debug ts=2026-05-16T12:14:16.255187949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T12:14:16.256843786Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T12:14:25.91832548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:14:28.046737884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:48868"
ts=2026-05-16T12:14:55.921706816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:15:01.487192184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:52482"
ts=2026-05-16T12:15:13.780839923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:60630"
level=debug ts=2026-05-16T12:15:17.377443093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:15:25.92446955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:15:43.783138879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:60816"
ts=2026-05-16T12:15:55.926077644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:16:01.491509466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:59238"
level=debug ts=2026-05-16T12:16:06.649170947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:16:25.928394418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:16:31.493401096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46304"
level=debug ts=2026-05-16T12:16:42.725277284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:16:55.930919818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
level=debug ts=2026-05-16T12:17:25.568323909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:17:25.934064377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:17:55.937149193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:17:58.067878366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:38320"
level=debug ts=2026-05-16T12:18:12.922872694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T12:18:12.92479799Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T12:18:13.794798287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:47058"
ts=2026-05-16T12:18:25.940041964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=debug ts=2026-05-16T12:18:53.818839955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:18:55.942716388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:19:01.504982954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:41882"
ts=2026-05-16T12:19:25.944871554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=debug ts=2026-05-16T12:19:53.761506677Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:19:55.947145568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:20:01.508390725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:45366"
ts=2026-05-16T12:20:13.804956731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:52054"
ts=2026-05-16T12:20:25.948900885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:20:28.084077166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:60310"
ts=2026-05-16T12:20:31.51009817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:41074"
level=debug ts=2026-05-16T12:20:43.413119902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:20:43.807172356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:44770"
ts=2026-05-16T12:20:55.950744938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:20:58.087503724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:59648"
ts=2026-05-16T12:21:13.809455943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:58578"
level=debug ts=2026-05-16T12:21:19.481188916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:21:25.953174602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:21:28.090441548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:59104"
ts=2026-05-16T12:21:31.513698481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:44254"
ts=2026-05-16T12:21:43.811170812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:57076"
ts=2026-05-16T12:21:55.956245954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:21:58.093809594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:51044"
level=debug ts=2026-05-16T12:21:59.817322908Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T12:21:59.81930744Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T12:22:09.419790313Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T12:22:09.42163385Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T12:22:09.423569351Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T12:22:09.423590185Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T12:22:25.959005598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:22:28.097063042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:59188"
level=debug ts=2026-05-16T12:22:46.502282428Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:22:55.961102578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:23:25.963410801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=debug ts=2026-05-16T12:23:40.45087215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:23:43.817949938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:54302"
ts=2026-05-16T12:23:55.965205207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:24:25.967776947Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:24:28.10864987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:38112"
ts=2026-05-16T12:24:31.529079467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:38128"
level=debug ts=2026-05-16T12:24:39.418199845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:24:55.970288101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:25:13.823344571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:57870"
ts=2026-05-16T12:25:25.973334676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:25:28.114418149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:34416"
ts=2026-05-16T12:25:31.533012784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:34838"
level=debug ts=2026-05-16T12:25:41.573143864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:25:55.976270066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:26:13.82664011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:57336"
ts=2026-05-16T12:26:25.979171806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=debug ts=2026-05-16T12:26:42.878957766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T12:26:42.880833816Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T12:26:43.829079265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:44522"
ts=2026-05-16T12:26:55.981833727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:26:58.123011687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:42048"
ts=2026-05-16T12:27:13.83161102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:36800"
ts=2026-05-16T12:27:25.983942356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=debug ts=2026-05-16T12:27:40.954891806Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:27:43.833388921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:39408"
ts=2026-05-16T12:27:55.986194035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:27:58.129584788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:43114"
level=debug ts=2026-05-16T12:28:22.535600513Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:28:25.987880875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:28:28.132577787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:57256"
ts=2026-05-16T12:28:55.990239845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=debug ts=2026-05-16T12:29:01.44450099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:29:01.543790804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42012"
ts=2026-05-16T12:29:25.992196471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:29:28.138491292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56354"
ts=2026-05-16T12:29:55.993712017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=debug ts=2026-05-16T12:29:57.277037781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:29:58.141238144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:57304"
ts=2026-05-16T12:30:01.548011042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:34708"
ts=2026-05-16T12:30:25.995900732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:30:55.998173986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
level=debug ts=2026-05-16T12:30:56.555626246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T12:30:56.557571722Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T12:31:26.000938312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:31:28.15029875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:55268"
ts=2026-05-16T12:31:56.00286979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=debug ts=2026-05-16T12:31:58.130546469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T12:32:09.420891651Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T12:32:09.422631908Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T12:32:09.424503862Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T12:32:09.424522778Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T12:32:26.004523202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:32:28.156502439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:57794"
ts=2026-05-16T12:32:31.559232159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:38198"
ts=2026-05-16T12:32:43.853368947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:47498"
level=debug ts=2026-05-16T12:32:44.160092889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:32:56.006579969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:33:26.009335615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=debug ts=2026-05-16T12:33:28.973620942Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:33:56.011251312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:33:58.165937711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:58752"
level=debug ts=2026-05-16T12:34:03.570179799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:34:26.012891223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:34:28.169347529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:36514"
level=debug ts=2026-05-16T12:34:55.619927448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T12:34:55.622156549Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T12:34:56.015496239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:35:26.018332715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=debug ts=2026-05-16T12:35:36.392703393Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:35:56.021245272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:36:01.575553951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:58654"
ts=2026-05-16T12:36:13.87017532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:46050"
ts=2026-05-16T12:36:26.024086733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:36:28.180885687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:52252"
level=debug ts=2026-05-16T12:36:32.464016506Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:36:56.026463526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:37:13.875387544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:56396"
level=debug ts=2026-05-16T12:37:24.214545872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:37:26.02961018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:37:31.581342262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:43730"
ts=2026-05-16T12:37:43.878019463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:55886"
ts=2026-05-16T12:37:56.032673874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:37:58.189739427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:33616"
level=debug ts=2026-05-16T12:38:09.38498541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:38:26.035109121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:38:28.193164766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:47206"
ts=2026-05-16T12:38:43.881332428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:39238"
level=debug ts=2026-05-16T12:38:50.41591651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T12:38:50.417837987Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T12:38:56.03699495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:39:01.587498686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:33270"
ts=2026-05-16T12:39:26.039150207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:39:28.198872936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:58446"
ts=2026-05-16T12:39:31.589570824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:37402"
level=debug ts=2026-05-16T12:39:37.637941381Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:39:43.88530389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:39950"
ts=2026-05-16T12:39:56.041461622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:40:13.887149448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:43582"
ts=2026-05-16T12:40:26.044654357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:40:28.204791216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56042"
ts=2026-05-16T12:40:31.592861753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:48838"
level=debug ts=2026-05-16T12:40:38.626320849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:40:43.888965238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:33544"
ts=2026-05-16T12:40:56.046986433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:41:26.049343221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
level=debug ts=2026-05-16T12:41:26.532955484Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:41:56.051289258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=error ts=2026-05-16T12:42:09.422107559Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T12:42:09.424067025Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T12:42:09.425911889Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-16T12:42:09.425933175Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-16T12:42:19.660625192Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:42:26.053473899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:42:31.599792326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56356"
ts=2026-05-16T12:42:56.055343353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=debug ts=2026-05-16T12:43:05.725315326Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T12:43:05.727026141Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T12:43:13.897657852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:53392"
ts=2026-05-16T12:43:26.057576168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:43:28.221576694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56864"
ts=2026-05-16T12:43:31.603579285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:47736"
level=debug ts=2026-05-16T12:43:47.137923792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:43:56.059993583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
level=debug ts=2026-05-16T12:44:23.935612105Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:44:26.062255728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:44:56.064533778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
level=debug ts=2026-05-16T12:45:15.272216975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:45:26.066800332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:45:28.23316522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:36866"
ts=2026-05-16T12:45:43.912822256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:57506"
level=debug ts=2026-05-16T12:45:56.064984509Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:45:56.068716079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:45:58.236845695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:57422"
ts=2026-05-16T12:46:26.071039824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:46:28.240153714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:53500"
level=debug ts=2026-05-16T12:46:34.607054408Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-16T12:46:34.608823258Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-16T12:46:43.916455269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:58686"
ts=2026-05-16T12:46:56.073348652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:46:58.243001094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:52282"
ts=2026-05-16T12:47:01.621302856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:48252"
level=debug ts=2026-05-16T12:47:22.674803404Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:47:26.075876112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdbec8e1 10.130.0.54:7946"
ts=2026-05-16T12:47:31.623621498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46122"
ts=2026-05-16T12:47:43.921944294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:53534"
ts=2026-05-16T12:47:56.078881738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:47:58.249394195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56160"
level=debug ts=2026-05-16T12:48:24.392400787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-16T12:48:26.08057304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pmqz2-ac315d4e 10.128.0.93:7946"
ts=2026-05-16T12:48:31.629375304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:34838"
ts=2026-05-16T12:48:56.0828749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-n97nk-36960196 10.128.0.90:7946"
ts=2026-05-16T12:48:58.255899957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:49516"
ts=2026-05-16T12:49:01.631595409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:54296"
level=debug ts=2026-05-16T12:49:02.874054119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"