level=info ts=2026-05-04T22:49:45.228404471Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-04T22:49:45.228469163Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-04T22:49:45.230845903Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-04T22:49:45.231239339Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-04T22:49:45.231361591Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.65 inf=eth0
level=debug ts=2026-05-04T22:49:45.231963152Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-04T22:49:45.232066743Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.65 inf=eth0
level=info ts=2026-05-04T22:49:45.232095781Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-b077c5d9
ts=2026-05-04T22:49:45.232203754Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-04T22:49:45.232215196Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.65 advertisePort=7946
level=debug ts=2026-05-04T22:49:45.23223052Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.65 advertisePort=7946
level=info ts=2026-05-04T22:49:45.232376605Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-04T22:49:45.239466699Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-04T22:49:45.239479895Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-04T22:49:45.239488312Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=7.114124ms
level=error ts=2026-05-04T22:49:45.239496303Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-04T22:49:45.239507025Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-04T22:49:45.242441961Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:49:45.242461067Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-04T22:49:45.247517992Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-05-04T22:49:45.247534098Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-04T22:49:45.247540762Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-04T22:49:45.247628954Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-04T22:49:45.24766927Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-04T22:49:45.247683117Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-04T22:49:45.247758096Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-05-04T22:49:45.247768927Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-04T22:49:45.247775216Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-04T22:49:45.248123749Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-04T22:49:45.248182205Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-04T22:49:45.248206398Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-04T22:49:45.24824042Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-04T22:49:45.248398165Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-04T22:49:45.374838133Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-04T22:49:45.374902378Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-04T22:49:46.375511981Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-04T22:49:46.511900653Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-04T22:49:46.511953739Z caller=loki.go:615 msg="Loki started" startup_time=1.351195938s
ts=2026-05-04T22:49:46.701150431Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:49:46.701174048Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:49:49.371570425Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:49:49.371597774Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-04T22:49:51.5124571Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-04T22:49:51.512511861Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-04T22:49:53.42658987Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T22:49:53.426612516Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-04T22:49:59.637883812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:50:06.059321527Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.63:7946: dial tcp 10.129.0.63:7946: connect: connection refused"
ts=2026-05-04T22:50:06.060644713Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.62:7946: dial tcp 10.129.0.62:7946: connect: connection refused"
ts=2026-05-04T22:50:06.061946428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.85:7946"
level=info ts=2026-05-04T22:50:06.06283226Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=20.823320601s
level=debug ts=2026-05-04T22:50:13.12788176Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:50:19.188195829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:54668"
ts=2026-05-04T22:50:24.667528034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T22:50:34.287488345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:50:45.134015719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:33438"
ts=2026-05-04T22:50:49.190396405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:60350"
ts=2026-05-04T22:50:54.669674819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T22:50:54.756637044Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:50:55.267449643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:50134"
ts=2026-05-04T22:51:15.135848817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49618"
level=debug ts=2026-05-04T22:51:22.694946969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-04T22:51:22.696786988Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T22:51:24.671056446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T22:51:54.672500896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T22:52:08.181655214Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:52:19.196422573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:59116"
ts=2026-05-04T22:52:24.673211944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T22:52:54.674780114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T22:52:55.272116057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:36484"
level=debug ts=2026-05-04T22:53:03.525312574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:53:15.141883377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48940"
ts=2026-05-04T22:53:19.200914574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:43708"
ts=2026-05-04T22:53:24.675746085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T22:53:45.143526673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48272"
ts=2026-05-04T22:53:49.203625966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:42482"
level=debug ts=2026-05-04T22:53:53.881987749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:53:54.677407389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T22:54:15.145082017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:53378"
ts=2026-05-04T22:54:24.679070458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T22:54:54.681189325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T22:54:55.278446058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:41264"
level=debug ts=2026-05-04T22:55:05.392599846Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T22:55:15.147833315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:36646"
ts=2026-05-04T22:55:24.683080392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T22:55:25.279808936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:58098"
ts=2026-05-04T22:55:49.212543884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:39814"
ts=2026-05-04T22:55:54.685221131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T22:55:55.281231222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:41614"
level=debug ts=2026-05-04T22:56:13.394278307Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-04T22:56:13.395848162Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T22:56:24.68704999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T22:56:54.689350206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T22:56:55.284581905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:48206"
level=debug ts=2026-05-04T22:56:59.969200726Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T22:57:24.691160592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T22:57:45.157199443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:39742"
ts=2026-05-04T22:57:54.692701647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T22:57:55.286951291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:45034"
level=debug ts=2026-05-04T22:58:00.3165064Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T22:58:15.159205866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:42220"
ts=2026-05-04T22:58:24.694096837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T22:58:45.160521781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:35474"
level=debug ts=2026-05-04T22:58:48.961844203Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T22:58:49.224026853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:39854"
ts=2026-05-04T22:58:54.695179288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T22:59:15.16179065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:44242"
ts=2026-05-04T22:59:24.696957576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T22:59:46.679190762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T22:59:49.228875822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:60878"
level=info ts=2026-05-04T22:59:51.512912787Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-04T22:59:51.517419903Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T22:59:51.51967134Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T22:59:51.522025589Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T22:59:51.522050418Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-04T22:59:51.522062405Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-04T22:59:54.699299535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:00:19.231225256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:37822"
ts=2026-05-04T23:00:24.701607178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:00:25.29670613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:59846"
level=debug ts=2026-05-04T23:00:30.273553691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:00:30.27515074Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:00:54.703180328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:01:24.705209393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:01:32.768566753Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:01:49.237714651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:41458"
ts=2026-05-04T23:01:54.707222609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:02:24.708968433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:02:25.303391682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:60830"
level=debug ts=2026-05-04T23:02:27.658409536Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:02:54.710669801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:02:55.304621419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:37800"
level=debug ts=2026-05-04T23:03:24.603074346Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:03:24.713065065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:03:45.177245381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:54668"
ts=2026-05-04T23:03:54.715399493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:03:55.308126493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:45408"
level=debug ts=2026-05-04T23:04:15.321655151Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:04:24.717404159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:04:49.249452209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:40636"
ts=2026-05-04T23:04:54.71974482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:05:11.696990331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:05:11.699523858Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:05:24.721803067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:05:45.184916603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49566"
level=debug ts=2026-05-04T23:05:53.893979184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:05:54.724933872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:06:24.726547206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:06:32.522888064Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:06:54.728275365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:06:55.31911211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:35158"
level=debug ts=2026-05-04T23:07:07.716054524Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:07:19.263559594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:57436"
ts=2026-05-04T23:07:24.730505534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:07:25.320340439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:54392"
ts=2026-05-04T23:07:45.192153398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:50272"
ts=2026-05-04T23:07:49.266129898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:50580"
level=debug ts=2026-05-04T23:07:54.20072864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:07:54.732553045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:08:24.733544822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:08:40.074204043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:08:40.075940333Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:08:54.737286635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:09:15.1964221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:43220"
ts=2026-05-04T23:09:24.742040316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:09:36.167562528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:09:49.276361067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53242"
level=error ts=2026-05-04T23:09:51.52776522Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:09:51.530131353Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:09:51.532213638Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:09:51.532241083Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:09:54.743476282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:10:15.199554552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:46292"
ts=2026-05-04T23:10:24.745133766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:10:33.510013485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:10:54.746078986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:10:55.330821343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:45098"
level=debug ts=2026-05-04T23:11:22.963519356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:11:24.747482934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:11:25.331974211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:33236"
ts=2026-05-04T23:11:54.749586172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:12:07.995553654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:12:24.751778467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:12:49.293174174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:37138"
ts=2026-05-04T23:12:54.753147358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:13:07.654518671Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:13:07.656133906Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:13:15.210119949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:46518"
ts=2026-05-04T23:13:24.754330884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:13:25.338540156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:37042"
level=debug ts=2026-05-04T23:13:44.550961117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:13:45.21123983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:33026"
ts=2026-05-04T23:13:54.756353155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:14:19.30249492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:34738"
ts=2026-05-04T23:14:24.758178755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:14:25.342160839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:49054"
level=debug ts=2026-05-04T23:14:32.221342106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:14:49.304863507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:51604"
ts=2026-05-04T23:14:54.760065633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:14:55.343707545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:55922"
ts=2026-05-04T23:15:19.306926465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:41062"
ts=2026-05-04T23:15:24.761499105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:15:28.951426837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:15:45.219595219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:33764"
ts=2026-05-04T23:15:49.308833097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:41052"
ts=2026-05-04T23:15:54.7635166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:16:15.525153109Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:16:19.310828968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:35668"
ts=2026-05-04T23:16:24.765690349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:16:25.349321232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:39352"
ts=2026-05-04T23:16:54.76754067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:17:09.581600531Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:17:09.583310434Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:17:24.769455375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:17:49.317424217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:59966"
ts=2026-05-04T23:17:54.771886095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:17:55.354026068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:48998"
level=debug ts=2026-05-04T23:18:10.616162358Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:18:24.773566273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:18:25.355099515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:55312"
ts=2026-05-04T23:18:54.776373427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:19:08.00714108Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:19:15.233543781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:55994"
ts=2026-05-04T23:19:24.778255412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:19:46.266995833Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:19:49.326379743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:60000"
level=error ts=2026-05-04T23:19:51.524545503Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:19:51.526409715Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:19:51.52849375Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:19:51.52851947Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:19:54.779644934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:19:55.359666122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:55662"
ts=2026-05-04T23:20:15.235806383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:53566"
ts=2026-05-04T23:20:24.781445198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:20:30.235096497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:20:54.782642084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:20:55.361886895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:49894"
level=debug ts=2026-05-04T23:21:11.522441818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:21:11.523955517Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:21:19.332344467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:33474"
ts=2026-05-04T23:21:24.784114806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:21:49.334957284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:58856"
ts=2026-05-04T23:21:54.786153742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:22:13.246740872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:22:15.241542334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:53988"
ts=2026-05-04T23:22:19.336467599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:58492"
ts=2026-05-04T23:22:24.787701224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:22:54.788754396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:23:13.024332299Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:23:19.340411286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38302"
ts=2026-05-04T23:23:24.79057852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:23:45.246344219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:44274"
ts=2026-05-04T23:23:49.342503019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:33664"
level=debug ts=2026-05-04T23:23:49.43754171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:23:54.792497221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:24:24.793874109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:24:36.585465317Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:24:45.250312272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:45688"
ts=2026-05-04T23:24:54.794984359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:25:15.251883369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:37886"
level=debug ts=2026-05-04T23:25:22.201429395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:25:22.203059373Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:25:24.796220011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:25:25.379071561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:56636"
ts=2026-05-04T23:25:45.252937227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:38142"
ts=2026-05-04T23:25:54.797838507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:26:03.418141646Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:26:15.254078084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:59526"
ts=2026-05-04T23:26:19.352615254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:35390"
ts=2026-05-04T23:26:24.79954606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:26:49.354587519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:43682"
ts=2026-05-04T23:26:54.801657461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:26:55.383792275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:57048"
level=debug ts=2026-05-04T23:26:58.918923217Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:27:24.806531699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:27:25.387388509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:35272"
ts=2026-05-04T23:27:45.258747433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:38348"
level=debug ts=2026-05-04T23:27:49.450110385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:27:54.81067038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:28:24.331817294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:28:24.815365571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:28:45.261080082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:54086"
ts=2026-05-04T23:28:54.817501496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:29:02.568146712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:29:02.570009069Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:29:24.819464901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:29:49.366404035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56852"
level=error ts=2026-05-04T23:29:51.527499752Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:29:51.529755148Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:29:51.531664387Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:29:51.531708205Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:29:54.821399541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:29:55.420101153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:57134"
level=debug ts=2026-05-04T23:30:02.180627605Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:30:24.823376603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:30:49.33633462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:30:49.37040367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:40948"
ts=2026-05-04T23:30:54.825120768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:31:24.826158514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:31:25.425005234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:58126"
level=debug ts=2026-05-04T23:31:27.559481625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:31:49.374631083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:45670"
ts=2026-05-04T23:31:54.828062247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:32:15.272200133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:39970"
ts=2026-05-04T23:32:24.829810691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:32:27.105321639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:32:45.273455065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:36926"
ts=2026-05-04T23:32:49.378739134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:40792"
ts=2026-05-04T23:32:54.830976505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:33:19.381129929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53624"
ts=2026-05-04T23:33:24.833247034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:33:25.015515656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:33:25.017066738Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:33:54.835511891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:33:55.436182037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:60506"
level=debug ts=2026-05-04T23:34:06.417302046Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:34:15.279361554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:41130"
ts=2026-05-04T23:34:24.836599276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:34:48.121155712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:34:49.38978997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:58340"
ts=2026-05-04T23:34:54.839045525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:35:19.39251385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38746"
ts=2026-05-04T23:35:24.840975014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:35:35.516426363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:35:54.842396146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:36:20.997220922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:36:24.843422132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:36:49.39925455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:45198"
ts=2026-05-04T23:36:54.845195019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:36:55.445907604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:34212"
level=debug ts=2026-05-04T23:37:07.693050578Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:37:07.694610066Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:37:19.401066347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:50950"
ts=2026-05-04T23:37:24.846524456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:37:25.447339523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:38290"
ts=2026-05-04T23:37:54.847521069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:38:00.268183466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:38:24.848563677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:38:45.295138143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49332"
level=debug ts=2026-05-04T23:38:48.088374614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:38:54.850365754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:38:55.452239954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:50716"
ts=2026-05-04T23:39:15.297068139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:56370"
ts=2026-05-04T23:39:19.409644655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:43366"
ts=2026-05-04T23:39:24.852173652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:39:25.453401694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:38338"
level=debug ts=2026-05-04T23:39:40.903521686Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:39:51.526185639Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:39:51.527812915Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:39:51.529401167Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:39:51.529419453Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:39:54.853641919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:40:24.855501309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:40:40.554949061Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:40:45.301915696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:56918"
ts=2026-05-04T23:40:49.417286464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:45780"
ts=2026-05-04T23:40:54.856814926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:40:55.45819265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:44690"
ts=2026-05-04T23:41:24.858387287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:41:30.86700087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:41:30.868475249Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:41:49.421988121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:43574"
ts=2026-05-04T23:41:54.860209116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:42:14.278910965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:42:24.862113358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:42:45.309535384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:47232"
ts=2026-05-04T23:42:54.863874098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:43:13.236104099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:43:15.311120321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:57428"
ts=2026-05-04T23:43:24.865183352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:43:25.466879452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:48652"
level=debug ts=2026-05-04T23:43:50.4963072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:43:54.86687128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:44:24.868377196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:44:25.470006873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:43730"
level=debug ts=2026-05-04T23:44:44.7528313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:44:45.315502976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:56606"
ts=2026-05-04T23:44:54.870571098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:44:55.471241293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:50790"
level=debug ts=2026-05-04T23:45:23.214692745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:45:23.216384013Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:45:24.872324993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:45:49.43928475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:59892"
ts=2026-05-04T23:45:54.874029831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:46:03.817881219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:46:24.87595162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:46:25.476125987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:40238"
level=debug ts=2026-05-04T23:46:50.669538863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:46:54.877319463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:46:55.478099326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:52564"
ts=2026-05-04T23:47:19.44562964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:36118"
ts=2026-05-04T23:47:24.879312736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:47:36.051455921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:47:49.448214705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:50420"
ts=2026-05-04T23:47:54.880825459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:48:19.450031692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:42406"
ts=2026-05-04T23:48:24.882751409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:48:35.403010543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:48:54.88438262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:49:13.932281807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:49:13.934235913Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:49:15.330995287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:42052"
ts=2026-05-04T23:49:24.886306288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:49:45.332810483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60390"
level=error ts=2026-05-04T23:49:51.526209656Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:49:51.528206414Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:49:51.530172975Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:49:51.530196535Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:49:54.888627626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:50:02.240349734Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:50:24.889721568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:50:25.490874409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:35562"
level=debug ts=2026-05-04T23:50:42.664864966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:50:54.891627989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:51:24.893376566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:51:34.518055674Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:51:49.464006169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56850"
ts=2026-05-04T23:51:54.895032429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:51:55.495294067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:34964"
ts=2026-05-04T23:52:19.466019296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:44530"
ts=2026-05-04T23:52:24.896686376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:52:27.490678207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:52:49.468351231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:55324"
ts=2026-05-04T23:52:54.898843812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:52:55.497957518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:53168"
level=debug ts=2026-05-04T23:53:07.653400444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:53:07.654836453Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:53:24.900305023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:53:49.472362526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:33976"
ts=2026-05-04T23:53:54.90174938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:54:07.920731819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:54:19.474042599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38788"
ts=2026-05-04T23:54:24.903678785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:54:49.475722919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:60930"
ts=2026-05-04T23:54:54.904960449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:54:55.504422252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:33212"
level=debug ts=2026-05-04T23:55:01.573660119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:55:24.906426581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-04T23:55:37.85807659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:55:45.352942464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:51988"
ts=2026-05-04T23:55:49.480491979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:51342"
ts=2026-05-04T23:55:54.907885751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:56:24.531514892Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:56:24.910113381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:56:25.510048496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:36688"
ts=2026-05-04T23:56:45.35615863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:45396"
ts=2026-05-04T23:56:54.911630715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:57:12.07427745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T23:57:12.075877374Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:57:19.485692557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:43964"
ts=2026-05-04T23:57:24.913150702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-04T23:57:54.915151925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-04T23:57:56.730916493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:58:15.361565605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49984"
ts=2026-05-04T23:58:19.4904043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:36558"
ts=2026-05-04T23:58:24.917056915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-04T23:58:38.065717488Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:58:49.492632501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:57326"
ts=2026-05-04T23:58:54.918248836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-04T23:59:15.366259002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:56090"
level=debug ts=2026-05-04T23:59:20.212711873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T23:59:24.920351504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=error ts=2026-05-04T23:59:51.526067253Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:59:51.527913064Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:59:51.529462685Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T23:59:51.529488515Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T23:59:54.922823245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-04T23:59:55.521559299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:36684"
level=debug ts=2026-05-05T00:00:07.772301014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:00:19.499249652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:46488"
ts=2026-05-05T00:00:24.924330582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-05T00:00:25.523483885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:57886"
ts=2026-05-05T00:00:45.371333448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:39132"
level=debug ts=2026-05-05T00:00:54.194639642Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:00:54.196302855Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:00:54.925777356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:01:19.504527125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:54276"
ts=2026-05-05T00:01:24.926944985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-05T00:01:30.791991419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:01:49.507192458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:51946"
ts=2026-05-05T00:01:54.929362208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-05T00:02:16.257278676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:02:24.930938321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:02:45.377785082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:47786"
ts=2026-05-05T00:02:54.933007285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-05T00:02:55.531610718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:53980"
level=debug ts=2026-05-05T00:03:09.778098647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:03:15.379512936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:34048"
ts=2026-05-05T00:03:24.935041493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:03:25.532593387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:43930"
ts=2026-05-05T00:03:54.936353397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-05T00:04:04.667538095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:04:24.938035185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:04:25.535322851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:33844"
ts=2026-05-05T00:04:54.939303321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-05T00:04:55.537133207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:39604"
level=debug ts=2026-05-05T00:05:02.48400717Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:05:02.485792137Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:05:15.383723842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:34884"
ts=2026-05-05T00:05:24.940553199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:05:45.385436431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:59002"
ts=2026-05-05T00:05:49.522963394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:59114"
level=debug ts=2026-05-05T00:05:54.036134069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:05:54.94181268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:06:24.942901535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:06:25.54208655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:48996"
level=debug ts=2026-05-05T00:06:45.814164585Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:06:54.944668446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:07:15.391644669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:42540"
ts=2026-05-05T00:07:24.946107261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:07:39.083785622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:07:54.947687115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:08:15.395390207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:51042"
ts=2026-05-05T00:08:24.948633187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:08:25.522279656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:08:25.550114861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:58260"
ts=2026-05-05T00:08:45.396945977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:33230"
ts=2026-05-05T00:08:54.950298236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:09:03.009418122Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:09:03.010889978Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:09:24.951779231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:09:41.12959251Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:09:45.400458024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:58156"
ts=2026-05-05T00:09:49.539414895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:39104"
level=error ts=2026-05-05T00:09:51.525221159Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T00:09:51.527659982Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T00:09:51.529363848Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T00:09:51.52938859Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:09:54.953334355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:10:24.954679329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:10:31.574868789Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:10:45.403651372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:55604"
ts=2026-05-05T00:10:54.956511943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-05T00:10:55.559631828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:40066"
level=debug ts=2026-05-05T00:11:09.356084374Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:11:15.40523312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:54718"
ts=2026-05-05T00:11:24.958618651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:11:43.635478402Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:11:54.960207003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:11:55.563840238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:52864"
ts=2026-05-05T00:12:24.961913943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-05T00:12:35.36797833Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:12:35.369658765Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:12:49.551052471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:43928"
ts=2026-05-05T00:12:54.963502518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-05T00:13:11.512778559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:13:15.411446886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:59456"
ts=2026-05-05T00:13:19.553365473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:46912"
ts=2026-05-05T00:13:24.965723012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-05T00:13:51.255059194Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:13:54.966765772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:13:55.571712493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:58014"
ts=2026-05-05T00:14:15.417517134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:40802"
ts=2026-05-05T00:14:24.967911128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:14:25.573501638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:45228"
level=debug ts=2026-05-05T00:14:44.274010905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:14:54.969699821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:14:55.575303748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:40392"
ts=2026-05-05T00:15:24.971239646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:15:25.576904165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:34842"
level=debug ts=2026-05-05T00:15:25.637319909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:15:45.421467201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:36358"
ts=2026-05-05T00:15:54.973232793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-05T00:15:55.578332318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:60214"
ts=2026-05-05T00:16:19.566120403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:42220"
level=debug ts=2026-05-05T00:16:24.460313534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:16:24.461981852Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:16:24.975005867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-05T00:16:49.568349086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:48652"
ts=2026-05-05T00:16:54.976799076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:17:15.425876072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:59014"
level=debug ts=2026-05-05T00:17:17.432221353Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:17:19.569883795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:47248"
ts=2026-05-05T00:17:24.979019773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-05T00:17:49.572016305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:50372"
ts=2026-05-05T00:17:54.980503197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
level=debug ts=2026-05-05T00:18:09.618439607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:18:24.982476402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-05T00:18:54.984092524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
level=debug ts=2026-05-05T00:19:04.857942696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:19:24.98543871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:19:49.580409482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:60986"
level=error ts=2026-05-05T00:19:51.526120792Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T00:19:51.52778726Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T00:19:51.529358861Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T00:19:51.529379834Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:19:54.986513953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:19:55.591258366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:56256"
level=debug ts=2026-05-05T00:19:58.295307932Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:20:24.988366567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:20:25.592701362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:54224"
level=debug ts=2026-05-05T00:20:45.663725073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:20:45.665509258Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:20:54.989810421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:20:55.593667351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:53202"
ts=2026-05-05T00:21:15.439313265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:41214"
ts=2026-05-05T00:21:24.990701754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:21:25.595392819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:44660"
level=debug ts=2026-05-05T00:21:37.545048033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:21:45.440599542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:56296"
ts=2026-05-05T00:21:54.991588866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:22:24.992719168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:22:25.598712643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:41308"
level=debug ts=2026-05-05T00:22:28.478781544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:22:49.592123906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53926"
ts=2026-05-05T00:22:54.99424205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-05T00:22:55.600067849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:52768"
level=debug ts=2026-05-05T00:23:09.403597832Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:23:24.995593793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:23:45.446196486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:51058"
ts=2026-05-05T00:23:49.596227081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:34124"
ts=2026-05-05T00:23:54.997118321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
level=debug ts=2026-05-05T00:24:09.827667805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:24:24.998777665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tt2kh-eb137c8f 10.130.0.64:7946"
ts=2026-05-05T00:24:49.600383836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:40294"
ts=2026-05-05T00:24:55.000357558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:24:55.606908785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:45156"
level=debug ts=2026-05-05T00:25:03.444931851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T00:25:03.446922994Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T00:25:25.00158948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bbf91f3f 10.130.0.66:7946"
ts=2026-05-05T00:25:45.451674915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48490"
level=debug ts=2026-05-05T00:25:50.475628481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T00:25:55.003394232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-626pb-c275cc1c 10.128.0.85:7946"
ts=2026-05-05T00:25:55.610403208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.66:52268"