level=info ts=2026-05-06T06:47:19.166031915Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-06T06:47:19.166064983Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-06T06:47:19.168166086Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-06T06:47:19.168678702Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-06T06:47:19.168781417Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.69 inf=eth0
level=debug ts=2026-05-06T06:47:19.170040991Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-06T06:47:19.170124751Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.69 inf=eth0
level=info ts=2026-05-06T06:47:19.17023659Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-73fd4119
ts=2026-05-06T06:47:19.170369997Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-06T06:47:19.170386866Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.69 advertisePort=7946
level=debug ts=2026-05-06T06:47:19.170408366Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.69 advertisePort=7946
level=info ts=2026-05-06T06:47:19.170596907Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-06T06:47:19.177792317Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-06T06:47:19.177810255Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-06T06:47:19.177826182Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=7.232002ms
level=error ts=2026-05-06T06:47:19.177838098Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-06T06:47:19.177851486Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-06T06:47:19.180311749Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T06:47:19.180335668Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-06T06:47:19.187995549Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-06T06:47:19.1880121Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-06T06:47:19.188033094Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-06T06:47:19.188065745Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-05-06T06:47:19.18807892Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-06T06:47:19.188096102Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-06T06:47:19.188178176Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-06T06:47:19.188191374Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-06T06:47:19.188384134Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-06T06:47:19.188395497Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-06T06:47:19.188455687Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-06T06:47:19.188443978Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-06T06:47:19.188467503Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-06T06:47:19.188874174Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-06T06:47:19.188886891Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-06T06:47:19.188899478Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-06T06:47:20.189980339Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-06T06:47:20.310216706Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-06T06:47:20.310286659Z caller=loki.go:615 msg="Loki started" startup_time=1.210826372s
ts=2026-05-06T06:47:20.767123625Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T06:47:20.767150695Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:47:24.731859566Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T06:47:24.73189123Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-06T06:47:25.310744442Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-06T06:47:25.310812954Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-06T06:47:30.70031161Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T06:47:30.700337924Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-06T06:47:32.272888136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:47:37.078853883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:44866"
ts=2026-05-06T06:47:40.43483585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.69:7946"
ts=2026-05-06T06:47:40.434942741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36272"
ts=2026-05-06T06:47:40.437059811Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.91:7946: dial tcp 10.128.0.91:7946: connect: connection refused"
ts=2026-05-06T06:47:40.438041998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.88:7946"
ts=2026-05-06T06:47:40.440226753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.89:7946"
level=info ts=2026-05-06T06:47:40.44132321Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=21.263465566s
ts=2026-05-06T06:47:44.554354031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:40572"
level=debug ts=2026-05-06T06:47:46.260098006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:47:46.470016215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:49874"
ts=2026-05-06T06:47:59.782160563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T06:48:06.827866749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:48:09.226935989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:57562"
ts=2026-05-06T06:48:29.784997034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T06:48:31.936260953Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:48:40.245757941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:42162"
ts=2026-05-06T06:48:59.788195712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T06:49:00.907098411Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-06T06:49:00.908812051Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T06:49:09.231255603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:40024"
ts=2026-05-06T06:49:10.248488414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:44350"
ts=2026-05-06T06:49:29.791487295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T06:49:36.675131369Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:49:59.794196181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T06:50:29.185054586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:57256"
ts=2026-05-06T06:50:29.797426917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T06:50:36.711177708Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:50:59.799909388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T06:51:27.131624245Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:51:29.190765326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:47772"
ts=2026-05-06T06:51:29.801627764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T06:51:39.243750848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:34062"
ts=2026-05-06T06:51:59.803307461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T06:52:09.24551943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:33238"
ts=2026-05-06T06:52:10.262452968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:60334"
level=debug ts=2026-05-06T06:52:27.881101064Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:52:29.805184912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T06:52:39.247112593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:46748"
ts=2026-05-06T06:52:59.807568604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T06:53:29.809546247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T06:53:33.652821739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-06T06:53:33.654410653Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T06:53:40.269453447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:45000"
ts=2026-05-06T06:53:59.810838637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T06:54:23.705022142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:54:29.812037659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T06:54:39.257202755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:43188"
ts=2026-05-06T06:54:40.275479375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:53624"
ts=2026-05-06T06:54:59.204691902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:51544"
ts=2026-05-06T06:54:59.814634172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T06:55:07.925271001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:55:09.259013043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:38480"
ts=2026-05-06T06:55:29.817145385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T06:55:54.94262263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:55:59.818114036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T06:56:29.212174242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:33720"
ts=2026-05-06T06:56:29.819697868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T06:56:39.081473813Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:56:40.283287054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:50594"
ts=2026-05-06T06:56:59.821397408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=info ts=2026-05-06T06:57:25.311314296Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-06T06:57:25.313707701Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T06:57:25.315482631Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T06:57:25.317402539Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T06:57:25.317437214Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-06T06:57:25.317452865Z caller=tables_manager.go:148 msg="compactor started"
level=debug ts=2026-05-06T06:57:25.777078767Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T06:57:25.778549815Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T06:57:29.823470219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T06:57:40.289242137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:60996"
ts=2026-05-06T06:57:59.825701082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T06:58:10.954091849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:58:29.826804882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T06:58:40.294546511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:56714"
ts=2026-05-06T06:58:59.223823463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:36034"
ts=2026-05-06T06:58:59.829452363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T06:59:08.676220793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:59:10.297009367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:34592"
ts=2026-05-06T06:59:29.83210331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T06:59:39.284004384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:33176"
level=debug ts=2026-05-06T06:59:47.496952835Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:59:59.835499821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:00:09.285095839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:57154"
ts=2026-05-06T07:00:29.840600691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:00:40.303420917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:45518"
level=debug ts=2026-05-06T07:00:48.369703712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:00:59.233366053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:57526"
ts=2026-05-06T07:00:59.841914866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T07:01:27.155907492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:01:27.157712424Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:01:29.844720038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:01:39.293381493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:58946"
ts=2026-05-06T07:01:59.238618918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:34630"
ts=2026-05-06T07:01:59.8480658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:02:09.295222689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:39460"
ts=2026-05-06T07:02:10.308810242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:48168"
level=debug ts=2026-05-06T07:02:26.480951964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:02:29.849910536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:02:39.296648543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:46334"
ts=2026-05-06T07:02:40.311603655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:50472"
ts=2026-05-06T07:02:59.242758271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:33786"
ts=2026-05-06T07:02:59.852489224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:03:09.298132921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:42258"
ts=2026-05-06T07:03:10.314152631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:39270"
level=debug ts=2026-05-06T07:03:18.459272797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:03:29.855367956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:03:40.318186818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:60240"
ts=2026-05-06T07:03:59.858369449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:04:12.237018851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:04:29.861040983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:04:39.304491412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:33674"
level=debug ts=2026-05-06T07:04:52.794765908Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:04:59.251629679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:48166"
ts=2026-05-06T07:04:59.862731923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:05:09.306638727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:57934"
ts=2026-05-06T07:05:29.254989598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:49522"
ts=2026-05-06T07:05:29.86507262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:05:39.308315266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:46372"
level=debug ts=2026-05-06T07:05:41.019084265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:05:41.02130525Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:05:59.257991782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:34824"
ts=2026-05-06T07:05:59.867851482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:06:29.261379778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:48846"
ts=2026-05-06T07:06:29.870755214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:06:39.312647348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:33306"
ts=2026-05-06T07:06:40.333563397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:53818"
level=debug ts=2026-05-06T07:06:40.386746033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:06:59.264108638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:41266"
ts=2026-05-06T07:06:59.873291371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:07:09.314310385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:60626"
ts=2026-05-06T07:07:10.337297729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:46782"
level=error ts=2026-05-06T07:07:25.322345972Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:07:25.324326797Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:07:25.326037645Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:07:25.326062024Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:07:29.875793895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:07:30.975950591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:07:59.877827916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T07:08:29.374588832Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:08:29.879998174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:08:40.345911378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:46966"
ts=2026-05-06T07:08:59.883130703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:09:22.081311291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:09:29.886026931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:09:40.352595678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:56532"
ts=2026-05-06T07:09:59.889332144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:10:08.688245085Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:10:08.689859315Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:10:10.355382743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:33124"
ts=2026-05-06T07:10:29.892547932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:10:40.363180691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:42064"
ts=2026-05-06T07:10:59.89508832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T07:11:01.156270004Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:11:29.898172699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:11:39.502706315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:11:59.292220905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:35648"
ts=2026-05-06T07:11:59.90176673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:12:10.370957316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:40176"
level=debug ts=2026-05-06T07:12:28.244712404Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:12:29.295329355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:53000"
ts=2026-05-06T07:12:29.905013533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:12:39.360323048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:56058"
ts=2026-05-06T07:12:59.908287394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:13:09.36530891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:59758"
level=debug ts=2026-05-06T07:13:12.088736996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:13:29.910258731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:13:59.91231759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:14:13.787660567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:14:13.789721179Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:14:29.30482968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:43772"
ts=2026-05-06T07:14:29.914483314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T07:14:55.32743627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:14:59.915559896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:15:10.385493376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:48894"
ts=2026-05-06T07:15:29.310451541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:42786"
ts=2026-05-06T07:15:29.918363704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T07:15:29.971438683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:15:40.388738795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:41060"
ts=2026-05-06T07:15:59.312939407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:42990"
ts=2026-05-06T07:15:59.920523946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:16:10.391267069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:44360"
ts=2026-05-06T07:16:29.319737242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:32836"
ts=2026-05-06T07:16:29.924097122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:16:31.762992646Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:16:39.38819242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:33152"
ts=2026-05-06T07:16:59.928263603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:17:09.389895352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:41482"
level=error ts=2026-05-06T07:17:25.320472998Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:17:25.323783985Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:17:25.325901166Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:17:25.325934855Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-06T07:17:25.378450713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:17:29.930565173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:17:40.397958482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:46642"
ts=2026-05-06T07:17:59.933264405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:18:13.177862171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:18:13.179836778Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:18:29.93596121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:18:59.938332737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T07:19:08.348807663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:19:29.940706453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:19:59.943635468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:20:08.338907209Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:20:09.405913866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:41452"
ts=2026-05-06T07:20:29.341544387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:58536"
ts=2026-05-06T07:20:29.945818391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:20:39.406999771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:54666"
ts=2026-05-06T07:20:40.410988265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:56764"
level=debug ts=2026-05-06T07:20:49.830509616Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:20:59.344255129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:48808"
ts=2026-05-06T07:20:59.947484174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:21:29.346741352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:51274"
ts=2026-05-06T07:21:29.948910537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T07:21:32.800337967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:21:59.950111244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:22:09.413409301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:39356"
level=debug ts=2026-05-06T07:22:29.917935196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:22:29.920035459Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:22:29.952272967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:22:39.414454028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:50632"
ts=2026-05-06T07:22:59.95499005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T07:23:22.717809041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:23:29.358942757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:36786"
ts=2026-05-06T07:23:29.957536147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:23:40.425417061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:53812"
ts=2026-05-06T07:23:59.361609528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:33550"
ts=2026-05-06T07:23:59.95973168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:24:04.469155339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:24:29.961466837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:24:59.36700846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:36158"
ts=2026-05-06T07:24:59.962446545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T07:25:00.857202489Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:25:09.42608175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:46844"
ts=2026-05-06T07:25:29.964385001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:25:39.428328416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:36668"
ts=2026-05-06T07:25:40.440752562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:32878"
level=debug ts=2026-05-06T07:25:55.175203619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:25:59.967120502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:26:10.444802822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:58784"
ts=2026-05-06T07:26:29.968911071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T07:26:54.993121859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:26:54.995081146Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:26:59.971543777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=error ts=2026-05-06T07:27:25.343861087Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:27:25.347303772Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:27:25.349315518Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:27:25.34934725Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:27:29.37887682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:46862"
ts=2026-05-06T07:27:29.97406849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T07:27:49.445295072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:27:59.976714948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:28:09.454450809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:43636"
ts=2026-05-06T07:28:29.383116304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:44308"
ts=2026-05-06T07:28:29.978596945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T07:28:50.961070231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:28:59.981243959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:29:09.459608743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:50252"
ts=2026-05-06T07:29:10.461203572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:50448"
ts=2026-05-06T07:29:29.388902046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:59050"
ts=2026-05-06T07:29:29.984102706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:29:39.461133487Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:48168"
ts=2026-05-06T07:29:40.464321631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:53964"
level=debug ts=2026-05-06T07:29:45.05320103Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:29:59.987103772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T07:30:23.391737101Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:30:29.393539117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:60664"
ts=2026-05-06T07:30:29.990671085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:30:39.465122207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:52984"
ts=2026-05-06T07:30:59.396534732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:33506"
ts=2026-05-06T07:30:59.994088557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:31:07.126797838Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:31:07.128393012Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:31:09.466549785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:53966"
ts=2026-05-06T07:31:10.470339555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:50326"
ts=2026-05-06T07:31:29.399868553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:49602"
ts=2026-05-06T07:31:29.996481069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:31:39.46802846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:50610"
level=debug ts=2026-05-06T07:31:49.291336968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:31:59.998604494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:32:24.137161419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:32:29.403951932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:34666"
ts=2026-05-06T07:32:30.001271723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:32:39.471832445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:60336"
ts=2026-05-06T07:33:00.00307121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T07:33:09.142343464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:33:30.004962185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:33:39.475967065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:58438"
ts=2026-05-06T07:33:40.482762107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:32968"
level=debug ts=2026-05-06T07:33:57.596713556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:34:00.008057223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:34:10.48590399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:51404"
ts=2026-05-06T07:34:29.41112402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:45772"
ts=2026-05-06T07:34:30.011092419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T07:34:47.70650944Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:34:47.708306905Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:35:00.013834035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:35:22.63553596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:35:30.017137329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:35:39.486731176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:54066"
ts=2026-05-06T07:36:00.019904604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:36:10.495052624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:46288"
level=debug ts=2026-05-06T07:36:20.347225882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:36:30.022061139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:36:39.491500475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:35418"
ts=2026-05-06T07:37:00.023746152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:37:09.492999605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:52762"
level=debug ts=2026-05-06T07:37:14.107401792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:37:25.321146665Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:37:25.322957858Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:37:25.324479837Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:37:25.324504413Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:37:29.424342121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:45794"
ts=2026-05-06T07:37:30.026645952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:37:39.494497884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:55026"
ts=2026-05-06T07:38:00.029651406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:38:02.062802709Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:38:30.03278711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:38:39.499863215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:51476"
level=debug ts=2026-05-06T07:38:41.413411109Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:38:41.415202317Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:38:59.43314438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:51292"
ts=2026-05-06T07:39:00.035088615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:39:09.501254552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:55986"
level=debug ts=2026-05-06T07:39:28.85495836Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:39:30.037649409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:40:00.040010769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:40:10.511778476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:38930"
level=debug ts=2026-05-06T07:40:18.194499046Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:40:30.042293553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:40:40.51497822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:52918"
ts=2026-05-06T07:40:59.447075406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:48736"
ts=2026-05-06T07:41:00.044875125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T07:41:06.67191008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:41:30.047461511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:41:39.514004803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:36982"
ts=2026-05-06T07:41:40.519377575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:43390"
ts=2026-05-06T07:41:59.451946148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:37970"
ts=2026-05-06T07:42:00.051252965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:42:07.831255737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:42:10.522049679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:59564"
ts=2026-05-06T07:42:30.053619238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:43:00.055737846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:43:05.844382763Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:43:05.846408434Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:43:09.521446802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:34306"
ts=2026-05-06T07:43:30.0583043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:43:39.522931946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:48264"
level=debug ts=2026-05-06T07:43:43.618748204Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:44:00.061356302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:44:10.530135556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:54642"
ts=2026-05-06T07:44:30.063664439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T07:44:43.14383448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:44:59.465586958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:44674"
ts=2026-05-06T07:45:00.065993888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:45:29.468437378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:59618"
ts=2026-05-06T07:45:30.068409508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T07:45:34.514478713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:45:40.537183878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:49858"
ts=2026-05-06T07:46:00.07018385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:46:09.535150776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:54422"
level=debug ts=2026-05-06T07:46:09.76475163Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:46:29.473612188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:48652"
ts=2026-05-06T07:46:30.072099914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:46:39.536990991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:55906"
level=debug ts=2026-05-06T07:46:56.793052586Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:46:56.79513213Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:46:59.476758916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:57354"
ts=2026-05-06T07:47:00.073824013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=error ts=2026-05-06T07:47:25.321831242Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:47:25.323946881Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:47:25.325977587Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:47:25.326005211Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:47:29.479456481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:39560"
ts=2026-05-06T07:47:30.076335549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:47:39.541092446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:34676"
level=debug ts=2026-05-06T07:47:42.978263394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:47:59.482177248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:45874"
ts=2026-05-06T07:48:00.079554789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:48:09.542823536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:48436"
level=debug ts=2026-05-06T07:48:29.497719196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:48:30.081952646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:48:59.486998716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:43760"
ts=2026-05-06T07:49:00.083875905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:49:30.08723386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:49:30.917926774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:49:39.548967287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:34064"
ts=2026-05-06T07:49:40.552895077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:60716"
ts=2026-05-06T07:50:00.090284773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:50:29.493144162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:43148"
ts=2026-05-06T07:50:30.092954244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:50:31.955983407Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:50:39.552967278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:58844"
ts=2026-05-06T07:50:59.496109166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:56218"
ts=2026-05-06T07:51:00.094952784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T07:51:19.821140364Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:51:19.823940528Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:51:30.09733114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:51:39.558001245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:41932"
ts=2026-05-06T07:52:00.10000475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:52:10.565784133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:51400"
level=debug ts=2026-05-06T07:52:20.653923187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:52:30.102799708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:52:39.561397119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:37614"
ts=2026-05-06T07:52:40.568077548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:55486"
ts=2026-05-06T07:52:59.504108058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:50602"
ts=2026-05-06T07:53:00.104954379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T07:53:18.861726645Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:53:29.50748802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:36440"
ts=2026-05-06T07:53:30.107024123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:53:57.054711934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:54:00.110277829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:54:30.11284436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T07:54:39.742474824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:55:00.115886755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T07:55:09.574048013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:35548"
ts=2026-05-06T07:55:10.577094309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:41040"
level=debug ts=2026-05-06T07:55:29.225063787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:55:29.226523547Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:55:30.118448288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:56:00.120543879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T07:56:27.835290383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:56:29.521930164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:35510"
ts=2026-05-06T07:56:30.123437679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:57:00.126396175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T07:57:07.321887087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:57:25.321963012Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:57:25.323885184Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:57:25.325895819Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:57:25.325916474Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:57:29.527093268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:56464"
ts=2026-05-06T07:57:30.128684747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T07:57:39.58605119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:55182"
ts=2026-05-06T07:58:00.131378623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T07:58:02.025698497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:58:29.532862025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:36214"
ts=2026-05-06T07:58:30.134272472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T07:58:59.535713902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:48132"
ts=2026-05-06T07:59:00.136947815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T07:59:03.509292768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:59:10.594196033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:55740"
ts=2026-05-06T07:59:30.139374226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T07:59:39.094431373Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:59:39.096213678Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:59:39.595080875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:58822"
ts=2026-05-06T08:00:00.141433985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T08:00:09.596112568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:40600"
level=debug ts=2026-05-06T08:00:23.707782664Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:00:30.143872688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T08:01:00.145991096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T08:01:25.267597871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:01:30.147715368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T08:02:00.150251979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T08:02:01.600870883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:02:10.610784881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:54520"
ts=2026-05-06T08:02:30.153249016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T08:02:40.614011758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:48020"
level=debug ts=2026-05-06T08:02:49.76648492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:02:59.550603732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:37160"
ts=2026-05-06T08:03:00.154947025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T08:03:30.157355151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T08:03:47.095002392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:03:47.096533828Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:04:00.160654796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T08:04:10.621092004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:53870"
ts=2026-05-06T08:04:30.163651878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T08:04:32.020659602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:05:00.166102867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T08:05:16.308107457Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:05:30.167902325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T08:05:39.627918354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:53186"
ts=2026-05-06T08:05:59.561258966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:60150"
ts=2026-05-06T08:06:00.169661443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T08:06:12.805135576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:06:30.171413398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T08:06:53.745174219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:07:00.173324215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=error ts=2026-05-06T08:07:25.322186671Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:07:25.32421138Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:07:25.326118088Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:07:25.326143508Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:07:30.175789676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T08:07:39.638235279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:55306"
level=debug ts=2026-05-06T08:07:50.832221985Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:07:50.833950663Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:08:00.178512572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T08:08:09.639541776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:38972"
ts=2026-05-06T08:08:10.638203459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:60524"
ts=2026-05-06T08:08:30.180224461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T08:08:31.791951129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:08:59.57337347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:47686"
ts=2026-05-06T08:09:00.181483238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T08:09:17.747554484Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:09:30.183620158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T08:09:52.611136419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:10:00.186242673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T08:10:30.189629148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T08:10:50.465562115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:11:00.192808718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T08:11:10.650914515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:44960"
ts=2026-05-06T08:11:29.58317408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:42782"
ts=2026-05-06T08:11:30.195174158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T08:11:40.627593195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:11:40.629171593Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:11:40.653853568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:51644"
ts=2026-05-06T08:12:00.197229076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T08:12:30.199321186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T08:12:30.748866723Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:12:39.664513293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:59960"
ts=2026-05-06T08:12:40.658754416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:58498"
ts=2026-05-06T08:13:00.201100812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
level=debug ts=2026-05-06T08:13:26.431414927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:13:30.203561541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T08:13:40.662720749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:49248"
ts=2026-05-06T08:14:00.206025105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T08:14:18.715147165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:14:29.596171423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:49932"
ts=2026-05-06T08:14:30.209221452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T08:14:39.674761803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:52650"
ts=2026-05-06T08:14:40.669215423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:40976"
ts=2026-05-06T08:15:00.211057444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T08:15:09.67621809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:59398"
level=debug ts=2026-05-06T08:15:16.094437957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:15:30.212864583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T08:15:40.675523594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:38772"
ts=2026-05-06T08:15:59.604999788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:52516"
ts=2026-05-06T08:16:00.215984758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T08:16:13.73823724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:16:13.740737168Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:16:30.217756656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T08:17:00.220007576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T08:17:09.251954177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:17:25.321904284Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:17:25.324063885Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:17:25.3260569Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:17:25.326077449Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:17:29.614115543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:39342"
ts=2026-05-06T08:17:30.222823582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T08:17:39.689235285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:56568"
ts=2026-05-06T08:17:40.681953073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:60668"
ts=2026-05-06T08:18:00.225405989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T08:18:07.630500299Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:18:10.684817037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:47652"
ts=2026-05-06T08:18:29.619448624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:51566"
ts=2026-05-06T08:18:30.227169619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T08:18:40.687351925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:35734"
level=debug ts=2026-05-06T08:18:56.250472182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:18:59.621634129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:45142"
ts=2026-05-06T08:19:00.228416895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T08:19:29.624734396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:39178"
ts=2026-05-06T08:19:30.230959037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T08:19:39.699290104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:56138"
level=debug ts=2026-05-06T08:19:49.801564863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:20:00.233356053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T08:20:09.701104755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:34214"
ts=2026-05-06T08:20:30.236236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T08:20:44.883147514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:20:44.885191673Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:21:00.239142904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T08:21:30.242191821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T08:21:33.491841878Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:22:00.24470421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f8553778 10.130.0.70:7946"
ts=2026-05-06T08:22:09.711450159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:39196"
level=debug ts=2026-05-06T08:22:27.89878416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:22:29.63380118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:59142"
ts=2026-05-06T08:22:30.247360826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T08:22:39.712931245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:41574"
ts=2026-05-06T08:23:00.250073961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
level=debug ts=2026-05-06T08:23:04.235614488Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:23:10.707705828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:44590"
ts=2026-05-06T08:23:29.639624079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:60540"
ts=2026-05-06T08:23:30.252823173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T08:23:59.276170433Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:23:59.642346948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:50806"
ts=2026-05-06T08:24:00.255676373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T08:24:10.713006492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:54036"
ts=2026-05-06T08:24:29.645654489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:60358"
ts=2026-05-06T08:24:30.257931698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T08:24:39.722349324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:59998"
level=debug ts=2026-05-06T08:24:42.760695583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:24:42.762508274Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:25:00.260657324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T08:25:10.716962944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:37990"
level=debug ts=2026-05-06T08:25:28.310417516Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:25:29.65010556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:49126"
ts=2026-05-06T08:25:30.263417135Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
ts=2026-05-06T08:25:59.653264143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:54680"
ts=2026-05-06T08:26:00.266689794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xpbpr-2971fa54 10.128.0.88:7946"
level=debug ts=2026-05-06T08:26:22.539258552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:26:29.656314573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:51760"
ts=2026-05-06T08:26:30.268968383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-spj28-8a14d7f2 10.128.0.89:7946"
ts=2026-05-06T08:26:39.732712029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:59096"