level=info ts=2026-05-13T20:47:33.676624869Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-13T20:47:33.676674455Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-13T20:47:33.678998097Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-13T20:47:33.679432933Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-13T20:47:33.679537499Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.69 inf=eth0
level=debug ts=2026-05-13T20:47:33.680025648Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-13T20:47:33.680108794Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.69 inf=eth0
level=info ts=2026-05-13T20:47:33.680109371Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-c0eeadc2
ts=2026-05-13T20:47:33.680219354Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-13T20:47:33.680233197Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.69 advertisePort=7946
level=debug ts=2026-05-13T20:47:33.680247476Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.69 advertisePort=7946
level=info ts=2026-05-13T20:47:33.680420939Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-13T20:47:33.686568126Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T20:47:33.686589088Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T20:47:33.686604074Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.184833ms
level=error ts=2026-05-13T20:47:33.686616325Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-13T20:47:33.686627172Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-13T20:47:33.68994213Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T20:47:33.689959836Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-13T20:47:33.696721398Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-13T20:47:33.69674254Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-13T20:47:33.696764507Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-13T20:47:33.696813803Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-13T20:47:33.696832156Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-05-13T20:47:33.696851452Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-13T20:47:33.696950956Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-13T20:47:33.696958393Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-13T20:47:33.697064441Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-13T20:47:33.697073762Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-13T20:47:33.697103438Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-13T20:47:33.697130307Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-13T20:47:33.697153002Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-13T20:47:33.697304804Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-13T20:47:33.697314178Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-13T20:47:33.697325079Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-13T20:47:34.698398997Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-13T20:47:34.83259317Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-13T20:47:34.83264614Z caller=loki.go:615 msg="Loki started" startup_time=1.227222534s
ts=2026-05-13T20:47:35.29960014Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T20:47:35.299625769Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:47:37.962369709Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T20:47:37.962397109Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-13T20:47:39.833108401Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-13T20:47:39.83316466Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-13T20:47:42.245727631Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T20:47:42.245756454Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:47:51.141768027Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.76:7946: dial tcp 10.129.0.76:7946: connect: connection refused"
ts=2026-05-13T20:47:51.143046266Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused"
ts=2026-05-13T20:47:51.143887478Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.66:7946: dial tcp 10.130.0.66:7946: connect: connection refused"
ts=2026-05-13T20:47:51.144439124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.67:7946"
ts=2026-05-13T20:47:51.14579383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.68:7946"
ts=2026-05-13T20:47:51.146304825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.69:7946"
ts=2026-05-13T20:47:51.146330366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49656"
level=info ts=2026-05-13T20:47:51.146573935Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=17.459940074s
level=debug ts=2026-05-13T20:47:52.19339538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:47:56.112224769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39500"
ts=2026-05-13T20:47:56.113480621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57786"
ts=2026-05-13T20:48:00.522281053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:39014"
ts=2026-05-13T20:48:04.117717694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39778"
ts=2026-05-13T20:48:07.977189631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:49310"
level=debug ts=2026-05-13T20:48:13.957810444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:48:15.783868055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T20:48:35.436283424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:48:45.785462465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T20:48:58.708139936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:49:04.125111981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39552"
ts=2026-05-13T20:49:15.786688334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T20:49:32.69030137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-13T20:49:32.692070823Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T20:49:34.128441772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34874"
ts=2026-05-13T20:49:45.788572048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T20:50:05.630076688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39240"
ts=2026-05-13T20:50:07.983944769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:46226"
level=debug ts=2026-05-13T20:50:13.208367792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:50:15.791067082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T20:50:34.134695534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34828"
ts=2026-05-13T20:50:35.63149017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:41154"
ts=2026-05-13T20:50:45.792006831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T20:51:04.137969103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43506"
level=debug ts=2026-05-13T20:51:15.660769502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:51:15.794160368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T20:51:34.140768354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43780"
ts=2026-05-13T20:51:45.794966327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T20:52:07.991742781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48488"
ts=2026-05-13T20:52:15.796556864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T20:52:17.933549868Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:52:45.798818551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T20:53:04.149472327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:37494"
level=debug ts=2026-05-13T20:53:05.068818406Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:53:07.993514295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:40904"
ts=2026-05-13T20:53:15.801368653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T20:53:45.803060816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T20:54:11.195857088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-13T20:54:11.197385247Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T20:54:15.805680997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T20:54:34.158679472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59450"
ts=2026-05-13T20:54:38.000669819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:46148"
ts=2026-05-13T20:54:45.808395468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T20:54:52.307918107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T20:55:04.161900771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:45882"
ts=2026-05-13T20:55:05.647396389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:43398"
ts=2026-05-13T20:55:08.002245233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45004"
ts=2026-05-13T20:55:15.810032407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T20:55:26.375489562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T20:55:34.164330239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:42276"
ts=2026-05-13T20:55:45.811638622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T20:56:04.167428495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:55084"
level=debug ts=2026-05-13T20:56:04.586983634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T20:56:15.81406354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T20:56:34.170538269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50936"
ts=2026-05-13T20:56:35.652862173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:41242"
ts=2026-05-13T20:56:38.006131515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48834"
ts=2026-05-13T20:56:45.816641279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T20:56:51.534905862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T20:57:04.173620154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54404"
ts=2026-05-13T20:57:15.82048809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T20:57:34.176779073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:37986"
ts=2026-05-13T20:57:35.655803971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42150"
ts=2026-05-13T20:57:38.010697842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:52594"
level=info ts=2026-05-13T20:57:39.834120991Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-13T20:57:39.837747681Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T20:57:39.839896534Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T20:57:39.841793206Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T20:57:39.841832307Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T20:57:39.841848183Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-13T20:57:45.822149784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T20:57:47.357172065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T20:57:47.35876727Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T20:58:04.179956811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:41446"
ts=2026-05-13T20:58:05.656871552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49006"
ts=2026-05-13T20:58:15.823649754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T20:58:27.229011386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T20:58:35.65770241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48578"
ts=2026-05-13T20:58:45.825906635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T20:59:04.186523378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46696"
ts=2026-05-13T20:59:15.829025107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T20:59:17.240530504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T20:59:45.832764499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T20:59:54.689822557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:00:15.835445252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T21:00:34.196505113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40588"
ts=2026-05-13T21:00:35.67028328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:40796"
ts=2026-05-13T21:00:45.838435019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:00:46.72426246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:01:04.199944313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:47910"
ts=2026-05-13T21:01:15.840336416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:01:32.983263183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:01:32.985319433Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:01:34.203410451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:35548"
ts=2026-05-13T21:01:45.842140369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:02:04.206508795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43058"
ts=2026-05-13T21:02:15.848647004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:02:24.618396716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:02:35.681026881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:50388"
ts=2026-05-13T21:02:45.851269351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:03:02.630442652Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:03:05.682290154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:46224"
ts=2026-05-13T21:03:15.853328636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:03:34.215977716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:58574"
ts=2026-05-13T21:03:38.039139493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35670"
ts=2026-05-13T21:03:45.854629898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:03:50.628413094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:04:04.219157901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:42036"
ts=2026-05-13T21:04:15.856078833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:04:34.221858645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43548"
ts=2026-05-13T21:04:35.688093798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:43476"
ts=2026-05-13T21:04:38.04171781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:39818"
ts=2026-05-13T21:04:45.857438693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:04:49.331593698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:05:15.858654643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:05:26.297991436Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:05:26.299528845Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:05:38.046271159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33846"
ts=2026-05-13T21:05:45.860027188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:06:04.231041522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43260"
ts=2026-05-13T21:06:08.047632001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:58310"
ts=2026-05-13T21:06:15.861562645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:06:25.005362622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:06:34.233882007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49514"
ts=2026-05-13T21:06:45.864335625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:07:15.234324437Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:07:15.866493326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:07:38.055077785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:55386"
level=error ts=2026-05-13T21:07:39.846383105Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:07:39.848673564Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:07:39.850931967Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:07:39.850970012Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:07:45.867938997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:07:49.989251864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:08:05.706983144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:34690"
ts=2026-05-13T21:08:15.87008895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T21:08:34.247085759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:55430"
level=debug ts=2026-05-13T21:08:36.423073526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:08:38.059196628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:36228"
ts=2026-05-13T21:08:45.87184443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:09:04.251490284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:37856"
ts=2026-05-13T21:09:15.873600576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:09:19.894681488Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:09:19.896282731Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:09:34.254930767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:38858"
ts=2026-05-13T21:09:35.714546749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:54822"
ts=2026-05-13T21:09:45.87540104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:10:04.107058285Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:10:04.258153128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:32880"
ts=2026-05-13T21:10:15.876685037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:10:35.717557752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:50940"
ts=2026-05-13T21:10:45.879222281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:10:55.222299966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:11:15.881613424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:11:34.271153719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:33078"
ts=2026-05-13T21:11:38.07108833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33778"
ts=2026-05-13T21:11:45.883131516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:11:55.877315395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:12:05.72131925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49500"
ts=2026-05-13T21:12:08.073234255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33364"
ts=2026-05-13T21:12:15.885053655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T21:12:34.277494381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57064"
ts=2026-05-13T21:12:38.075811734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51778"
ts=2026-05-13T21:12:45.887088674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:12:56.585662226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:13:04.280673447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54098"
ts=2026-05-13T21:13:15.888081517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:13:36.100097435Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:13:36.102309785Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:13:38.07806938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59534"
ts=2026-05-13T21:13:45.890420857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T21:14:04.286639045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40126"
ts=2026-05-13T21:14:15.892859309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:14:17.098725593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:14:34.289512126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54132"
ts=2026-05-13T21:14:45.895149407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:15:07.723380667Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:15:15.898463281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:15:35.731004867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:55448"
ts=2026-05-13T21:15:38.08952031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:43516"
ts=2026-05-13T21:15:45.901136232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:15:47.287794103Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:16:15.90376199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:16:34.300702533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48884"
level=debug ts=2026-05-13T21:16:43.371624698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:16:45.905138184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:17:04.303706152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:58618"
ts=2026-05-13T21:17:15.907169664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:17:24.702925161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:17:24.704479059Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:17:38.097743487Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:52544"
level=error ts=2026-05-13T21:17:39.845670885Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:17:39.847442731Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:17:39.849279402Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:17:39.849303301Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:17:45.908841733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:18:03.025376318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:18:04.310062781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:41202"
ts=2026-05-13T21:18:05.736717898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47812"
ts=2026-05-13T21:18:15.910704877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T21:18:34.318483779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49014"
ts=2026-05-13T21:18:45.914122041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:18:58.500531342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:19:05.740131116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48450"
ts=2026-05-13T21:19:08.105724255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:58354"
ts=2026-05-13T21:19:15.915824922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:19:35.741854112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49134"
ts=2026-05-13T21:19:45.918265945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:20:00.726318678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:20:15.920225185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:20:45.921144737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:20:56.147601534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:21:15.921973193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:21:36.900332824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:21:36.901890732Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:21:45.924364687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T21:22:04.340694277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46466"
ts=2026-05-13T21:22:15.927814385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:22:33.692715582Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:22:34.343471456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39088"
ts=2026-05-13T21:22:38.119842033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:41984"
ts=2026-05-13T21:22:45.929674931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:23:15.930975124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:23:22.038198544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:23:34.349191772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34838"
ts=2026-05-13T21:23:45.933251141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:24:05.318314995Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:24:05.761203343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:51222"
ts=2026-05-13T21:24:08.126720147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57736"
ts=2026-05-13T21:24:15.935528617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:24:35.762189816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:55526"
level=debug ts=2026-05-13T21:24:45.582701686Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:24:45.93688061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:25:04.35839444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54242"
ts=2026-05-13T21:25:15.937987963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:25:34.361784335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60030"
level=debug ts=2026-05-13T21:25:39.487970171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:25:39.489970064Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:25:45.939890667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:26:04.364776381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60070"
ts=2026-05-13T21:26:05.769288379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48888"
ts=2026-05-13T21:26:15.942108614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T21:26:35.770457622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:54650"
level=debug ts=2026-05-13T21:26:36.562643187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:26:45.944277269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:27:05.771264906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47470"
ts=2026-05-13T21:27:08.137807706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57092"
ts=2026-05-13T21:27:15.945897344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:27:24.118396425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:27:38.139405058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:40626"
level=error ts=2026-05-13T21:27:39.8480413Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:27:39.850105108Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:27:39.851969735Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:27:39.851991136Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:27:45.947506265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:28:07.692424859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:28:08.140431471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:41950"
ts=2026-05-13T21:28:15.949239847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:28:45.950706227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:29:01.353398688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:29:04.383806425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40672"
ts=2026-05-13T21:29:08.145322476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:53356"
ts=2026-05-13T21:29:15.951653296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:29:36.196483002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:29:36.198047967Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:29:45.952878759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:30:05.782298221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:36180"
ts=2026-05-13T21:30:08.147902763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:38632"
level=debug ts=2026-05-13T21:30:12.988998969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:30:15.953910859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:30:38.149565151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51336"
ts=2026-05-13T21:30:45.956444894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:31:08.568096687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:31:15.958177104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:31:42.622254718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:31:45.959839667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:32:08.153517977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:38230"
ts=2026-05-13T21:32:15.961458225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:32:23.673411765Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:32:45.9633133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:33:04.407598209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:51992"
ts=2026-05-13T21:33:15.964370168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:33:22.587227962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:33:22.588939569Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:33:34.410435466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:37700"
ts=2026-05-13T21:33:45.96609201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:34:15.967527753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:34:19.147863314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:34:34.416771914Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43508"
ts=2026-05-13T21:34:45.969560941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:35:05.804778752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:44958"
level=debug ts=2026-05-13T21:35:10.098754614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:35:15.971804134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T21:35:34.42325503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:36830"
ts=2026-05-13T21:35:45.973861526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:35:46.385891757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:36:08.171360985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:41004"
ts=2026-05-13T21:36:15.976585407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T21:36:34.428991542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60688"
ts=2026-05-13T21:36:35.809358499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:45358"
ts=2026-05-13T21:36:45.978224297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:36:46.64102702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:37:05.810627312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:54410"
ts=2026-05-13T21:37:15.980223835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T21:37:34.434897403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:42092"
level=debug ts=2026-05-13T21:37:35.580536605Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:37:35.582944082Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:37:35.811899074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:43396"
level=error ts=2026-05-13T21:37:39.844606234Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:37:39.846284009Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:37:39.847842654Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:37:39.847863849Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:37:45.982083617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:38:13.249589262Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:38:15.984924774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T21:38:45.986931579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:38:50.5680213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:39:04.443964222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49310"
ts=2026-05-13T21:39:05.816419313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57078"
ts=2026-05-13T21:39:15.989125633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:39:24.783542334Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:39:35.818182612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:56448"
ts=2026-05-13T21:39:45.992625936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T21:40:08.187366485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33976"
level=debug ts=2026-05-13T21:40:10.004143475Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:40:15.99436729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:40:35.821020361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:54954"
ts=2026-05-13T21:40:45.995546994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:41:04.456480058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59800"
level=debug ts=2026-05-13T21:41:08.551670659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:41:08.554265536Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:41:15.997060261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:41:35.823909408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:54710"
level=debug ts=2026-05-13T21:41:45.200764565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:41:45.99829693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:42:05.824947218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:36240"
ts=2026-05-13T21:42:08.19471427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:39848"
ts=2026-05-13T21:42:16.000215974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:42:34.465334569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50794"
level=debug ts=2026-05-13T21:42:42.824216028Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:42:46.002672183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T21:43:16.004851782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:43:35.830035629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:33454"
level=debug ts=2026-05-13T21:43:37.294501264Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:43:46.007125133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T21:44:08.203254776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33230"
ts=2026-05-13T21:44:16.009431654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:44:29.551610558Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:44:46.01055776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:45:04.481138527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40662"
ts=2026-05-13T21:45:16.011592913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:45:25.198849659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:45:25.20055966Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:45:46.012740966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:46:08.212212294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:43120"
level=debug ts=2026-05-13T21:46:11.71226848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:46:16.014043509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:46:34.490568581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:38308"
ts=2026-05-13T21:46:35.841214165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:43372"
ts=2026-05-13T21:46:46.015895653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:47:08.215638067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:40604"
level=debug ts=2026-05-13T21:47:10.864761485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:47:16.017246772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=error ts=2026-05-13T21:47:39.848010541Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:47:39.849953395Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:47:39.852216259Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:47:39.852246705Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:47:46.018284854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:47:56.992011933Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:48:16.019820665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:48:46.022540384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:48:52.651323442Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:49:16.025752656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T21:49:38.224191384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60118"
level=debug ts=2026-05-13T21:49:43.774417938Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:49:43.776817012Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:49:46.027743002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:50:04.512313408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:53460"
ts=2026-05-13T21:50:05.858277834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39966"
ts=2026-05-13T21:50:16.029316827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:50:26.493945364Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:50:34.515276794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39958"
ts=2026-05-13T21:50:46.030561868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:51:05.863434194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:33572"
ts=2026-05-13T21:51:16.032751346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:51:26.095963971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:51:34.521431371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:37346"
ts=2026-05-13T21:51:35.865108025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49848"
ts=2026-05-13T21:51:46.034686688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:52:16.037429363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:52:16.775436171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:52:38.233800538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:34224"
ts=2026-05-13T21:52:46.040457562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T21:52:51.01898469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:53:04.530471063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49844"
ts=2026-05-13T21:53:08.235287617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35276"
ts=2026-05-13T21:53:16.042395091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:53:33.129732087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:53:33.131306596Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:53:46.044076158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T21:54:05.875120685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:35868"
level=debug ts=2026-05-13T21:54:11.268585709Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:54:16.046121607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:54:34.538389223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:42436"
ts=2026-05-13T21:54:46.047089087Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:54:53.952787737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:55:16.048626476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:55:34.241859094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:55:35.880639595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:53940"
ts=2026-05-13T21:55:46.051126212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T21:56:04.547224138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34056"
ts=2026-05-13T21:56:16.053879181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:56:23.729596941Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:56:35.885820797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:43018"
ts=2026-05-13T21:56:46.055053957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:57:08.2517545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:54914"
ts=2026-05-13T21:57:16.056619414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:57:23.436545012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:57:23.438079501Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:57:39.846561676Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:57:39.848542083Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:57:39.850385562Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:57:39.850406661Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:57:46.058315113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:58:16.059888575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:58:19.486306767Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:58:34.562565574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:42314"
ts=2026-05-13T21:58:35.893307048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47256"
ts=2026-05-13T21:58:38.255551415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:43250"
ts=2026-05-13T21:58:46.061617241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:59:11.553513737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:59:16.063292385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:59:34.568629959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:55410"
ts=2026-05-13T21:59:46.064216861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:00:03.710746286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:00:05.898595328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57534"
ts=2026-05-13T22:00:16.065048967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T22:00:35.900429386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:41526"
ts=2026-05-13T22:00:38.265207041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:52718"
ts=2026-05-13T22:00:46.066031985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:01:03.684921007Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:01:04.578988669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:47696"
ts=2026-05-13T22:01:08.266750676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48236"
ts=2026-05-13T22:01:16.067160887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T22:01:46.068197884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T22:02:04.693302601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:02:04.694815252Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:02:08.26930191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48946"
ts=2026-05-13T22:02:16.069606587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T22:02:34.588295839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59874"
ts=2026-05-13T22:02:38.271247197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60646"
level=debug ts=2026-05-13T22:02:43.758717547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:02:46.070873817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T22:03:16.073163027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T22:03:34.779373807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:03:38.274971174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:49370"
ts=2026-05-13T22:03:46.075737768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T22:04:05.916682504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:60338"
ts=2026-05-13T22:04:16.076798435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:04:31.896125412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:04:46.078720185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T22:05:08.282310719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:53708"
ts=2026-05-13T22:05:16.080742454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T22:05:19.182747068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:05:46.081582849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T22:06:05.924311319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:33642"
level=debug ts=2026-05-13T22:06:11.390652104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:06:11.392085498Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:06:16.083370064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T22:06:34.613704953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:53038"
ts=2026-05-13T22:06:35.925983919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:46958"
ts=2026-05-13T22:06:46.084447806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:07:00.199079909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:07:05.927551157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:38498"
ts=2026-05-13T22:07:16.085276361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T22:07:34.620137712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48364"
ts=2026-05-13T22:07:38.29248652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59330"
level=error ts=2026-05-13T22:07:39.848892994Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:07:39.850817762Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:07:39.852472506Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:07:39.852492832Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T22:07:43.301841916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:07:46.08683913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T22:08:04.623445929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49754"
ts=2026-05-13T22:08:05.931200058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:44570"
ts=2026-05-13T22:08:16.088159843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T22:08:23.469675192Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:08:38.296476096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:53994"
ts=2026-05-13T22:08:46.089509375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T22:09:05.933609904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:38788"
level=debug ts=2026-05-13T22:09:06.620780136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:09:08.29791431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47338"
ts=2026-05-13T22:09:16.091507178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T22:09:34.633132647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49266"
ts=2026-05-13T22:09:38.299488679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45632"
level=debug ts=2026-05-13T22:09:45.878923754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:09:45.880779444Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:09:46.093576416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T22:10:16.095760861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T22:10:46.097387219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:10:46.360998494Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:11:08.306046286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60856"
ts=2026-05-13T22:11:16.099523702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T22:11:46.101995228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:11:46.513209153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:12:04.648810659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:52168"
ts=2026-05-13T22:12:08.310166485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:43422"
ts=2026-05-13T22:12:16.103002209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T22:12:26.194029397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:12:46.10428032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T22:13:04.654617126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:42908"
ts=2026-05-13T22:13:05.951671239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:40228"
level=debug ts=2026-05-13T22:13:11.404201903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:13:16.105588825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T22:13:35.952780011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57244"
ts=2026-05-13T22:13:46.107454915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T22:14:01.918938979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:14:01.92043279Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:14:04.661066199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46858"
ts=2026-05-13T22:14:05.954333878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49954"
ts=2026-05-13T22:14:08.320021166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59624"
ts=2026-05-13T22:14:16.111003943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T22:14:46.11371066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:15:02.873415577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:15:08.324176926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:37100"
ts=2026-05-13T22:15:16.114975909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T22:15:38.325856316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:46384"
level=debug ts=2026-05-13T22:15:41.885622024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:15:46.116602313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T22:16:04.673036512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46510"
ts=2026-05-13T22:16:08.327115343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:38406"
ts=2026-05-13T22:16:16.11973232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T22:16:29.639882216Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:16:34.676524623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:38750"
ts=2026-05-13T22:16:35.965246489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:53602"
ts=2026-05-13T22:16:46.121683866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T22:17:08.329398575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:37890"
ts=2026-05-13T22:17:16.124081942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T22:17:16.148927688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:17:39.846405001Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:17:39.848098986Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:17:39.849848477Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:17:39.849873417Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:17:46.12662728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:17:54.251497224Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:17:54.253185708Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:18:05.969514811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:34846"
ts=2026-05-13T22:18:16.128461448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T22:18:34.688350228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57920"
ts=2026-05-13T22:18:38.332957308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57938"
ts=2026-05-13T22:18:46.130179785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T22:18:47.516643681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:19:05.972272153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:46304"
ts=2026-05-13T22:19:08.333883212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:49828"
ts=2026-05-13T22:19:16.132117762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T22:19:34.696213166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46552"
ts=2026-05-13T22:19:35.973669353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42408"
level=debug ts=2026-05-13T22:19:37.96048079Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:19:38.335031296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48362"
ts=2026-05-13T22:19:46.135251619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T22:20:04.698633031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57708"
ts=2026-05-13T22:20:16.137680981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:20:19.686238778Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:20:38.337561335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:38784"
ts=2026-05-13T22:20:46.139553367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T22:21:04.704970735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57668"
ts=2026-05-13T22:21:08.339011566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:39668"
ts=2026-05-13T22:21:16.141053715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T22:21:20.946072826Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:21:38.340766813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:56960"
ts=2026-05-13T22:21:46.142237141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T22:22:16.143697629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T22:22:22.466136354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:22:22.467663568Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:22:34.714118156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:47562"
ts=2026-05-13T22:22:38.345484233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35282"
ts=2026-05-13T22:22:46.145517716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
level=debug ts=2026-05-13T22:23:12.552085212Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:23:16.147325745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T22:23:35.989803917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:40310"
ts=2026-05-13T22:23:46.149096081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:23:57.098650027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:24:16.150297585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:24:34.399940199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:24:38.354984147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:41676"
ts=2026-05-13T22:24:46.152872223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76d5aec9 10.129.0.75:7946"
ts=2026-05-13T22:25:08.35626214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:53442"
ts=2026-05-13T22:25:16.154676702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T22:25:18.708734948Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"