level=info ts=2026-05-09T02:42:38.408965039Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)" level=info ts=2026-05-09T02:42:38.409006804Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml level=info ts=2026-05-09T02:42:38.41142471Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095 level=debug ts=2026-05-09T02:42:38.411951852Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-05-09T02:42:38.412049125Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.81 inf=eth0 level=debug ts=2026-05-09T02:42:38.41282643Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-05-09T02:42:38.412919909Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.81 inf=eth0 level=info ts=2026-05-09T02:42:38.413000602Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-24c4e363 ts=2026-05-09T02:42:38.413093943Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired" level=debug ts=2026-05-09T02:42:38.413123036Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.81 advertisePort=7946 level=debug ts=2026-05-09T02:42:38.413138167Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.81 advertisePort=7946 level=info ts=2026-05-09T02:42:38.413285445Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4 ts=2026-05-09T02:42:38.418910761Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=info ts=2026-05-09T02:42:38.418927477Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" level=warn ts=2026-05-09T02:42:38.418937629Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.654259ms level=error ts=2026-05-09T02:42:38.418948833Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1" level=info ts=2026-05-09T02:42:38.418960196Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local ts=2026-05-09T02:42:38.42155062Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-05-09T02:42:38.421567401Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" level=info ts=2026-05-09T02:42:38.433942803Z caller=module_service.go:82 msg=starting module=runtime-config level=debug ts=2026-05-09T02:42:38.433996983Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server level=info ts=2026-05-09T02:42:38.434065607Z caller=module_service.go:82 msg=starting module=server level=debug ts=2026-05-09T02:42:38.434095658Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config level=info ts=2026-05-09T02:42:38.434135709Z caller=module_service.go:82 msg=starting module=analytics level=info ts=2026-05-09T02:42:38.434161029Z caller=module_service.go:82 msg=starting module=memberlist-kv level=debug ts=2026-05-09T02:42:38.434231006Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server level=debug ts=2026-05-09T02:42:38.434242289Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics level=debug ts=2026-05-09T02:42:38.434266471Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv level=info ts=2026-05-09T02:42:38.434279192Z caller=module_service.go:82 msg=starting module=compactor level=info ts=2026-05-09T02:42:38.434328303Z caller=ring.go:390 msg="ring doesn't exist in KV store yet" level=info ts=2026-05-09T02:42:38.434355458Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor level=info ts=2026-05-09T02:42:38.434371667Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty" level=info ts=2026-05-09T02:42:38.434482143Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring" level=info ts=2026-05-09T02:42:38.434491585Z caller=compactor.go:416 msg="compactor is JOINING in the ring" level=debug ts=2026-05-09T02:42:38.434501066Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true level=info ts=2026-05-09T02:42:39.435638291Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring" level=info ts=2026-05-09T02:42:39.435714147Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring" level=info ts=2026-05-09T02:42:39.435744068Z caller=loki.go:615 msg="Loki started" startup_time=1.124266469s ts=2026-05-09T02:42:39.623381297Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-05-09T02:42:39.62340366Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" ts=2026-05-09T02:42:42.305814247Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-05-09T02:42:42.305837026Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" level=info ts=2026-05-09T02:42:44.436518908Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor" level=info ts=2026-05-09T02:42:44.436582012Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor" ts=2026-05-09T02:42:47.404943935Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-05-09T02:42:47.404970661Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" ts=2026-05-09T02:42:57.768992783Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.70:7946: dial tcp 10.130.0.70:7946: connect: connection refused" ts=2026-05-09T02:42:57.770239418Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.72:7946: dial tcp 10.130.0.72:7946: connect: connection refused" ts=2026-05-09T02:42:57.771144532Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.71:7946: dial tcp 10.130.0.71:7946: connect: connection refused" level=warn ts=2026-05-09T02:42:57.771151159Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="3 errors occurred:\n\t* failed to join 10.130.0.70:7946: dial tcp 10.130.0.70:7946: connect: connection refused\n\t* failed to join 10.130.0.72:7946: dial tcp 10.130.0.72:7946: connect: connection refused\n\t* failed to join 10.130.0.71:7946: dial tcp 10.130.0.71:7946: connect: connection refused\n\n" level=debug ts=2026-05-09T02:42:59.952417222Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-05-09T02:43:03.643563988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:53254" ts=2026-05-09T02:43:05.286069994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:55658" ts=2026-05-09T02:43:09.154194718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:53338" ts=2026-05-09T02:43:11.227558067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T02:43:15.626594719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-05-09T02:43:18.305471855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:44460" ts=2026-05-09T02:43:21.829294285Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.70:7946: dial tcp 10.130.0.70:7946: connect: connection refused" ts=2026-05-09T02:43:21.829841713Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.72:7946: dial tcp 10.130.0.72:7946: connect: connection refused" ts=2026-05-09T02:43:21.830376865Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.71:7946: dial tcp 10.130.0.71:7946: connect: connection refused" ts=2026-05-09T02:43:21.830501555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.128.0.76:7946" ts=2026-05-09T02:43:21.831156304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.128.0.81:7946" ts=2026-05-09T02:43:21.831183722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:36200" ts=2026-05-09T02:43:21.831634034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.128.0.77:7946" level=info ts=2026-05-09T02:43:21.832426135Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=43.413459615s ts=2026-05-09T02:43:28.800798595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:33208" level=debug ts=2026-05-09T02:43:38.060482271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-05-09T02:43:41.229282915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T02:44:00.622276441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-05-09T02:44:11.230151272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T02:44:35.930353016Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=error ts=2026-05-09T02:44:35.931966382Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T02:44:41.231119759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T02:44:58.804057554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:55386" ts=2026-05-09T02:45:11.232030806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T02:45:15.316961922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:55628" level=debug ts=2026-05-09T02:45:18.40964286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-05-09T02:45:41.233192384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T02:45:45.31852307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:36580" ts=2026-05-09T02:46:11.234593773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T02:46:15.319973984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:42902" level=debug ts=2026-05-09T02:46:25.499035904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-05-09T02:46:28.807826193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:59900" ts=2026-05-09T02:46:41.236303507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T02:47:11.237508052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T02:47:26.895511324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-05-09T02:47:41.238451282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T02:47:58.812198932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:39450" ts=2026-05-09T02:48:09.168261639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:35150" ts=2026-05-09T02:48:11.240702826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T02:48:15.325413322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:48276" level=debug ts=2026-05-09T02:48:35.450997369Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-05-09T02:48:41.241703316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T02:49:09.170853666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:40078" ts=2026-05-09T02:49:11.242431228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T02:49:28.815393881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:43804" ts=2026-05-09T02:49:41.244064637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T02:49:49.331840456Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=error ts=2026-05-09T02:49:49.333499474Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T02:50:11.245207975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T02:50:41.246789476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T02:50:50.391949165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T02:51:09.176471148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:59254" ts=2026-05-09T02:51:11.247813028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T02:51:39.178143521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:48288" ts=2026-05-09T02:51:41.248829461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T02:51:45.33444838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:59888" level=debug ts=2026-05-09T02:51:51.727101998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T02:51:58.821542882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:35732" ts=2026-05-09T02:52:11.250190122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T02:52:28.711452033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T02:52:28.822917417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:55844" ts=2026-05-09T02:52:41.252002845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=info ts=2026-05-09T02:52:44.436608428Z caller=tables_manager.go:76 msg="compactor startup delay completed" level=error ts=2026-05-09T02:52:44.440424638Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T02:52:44.442399729Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T02:52:44.444378178Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T02:52:44.444403248Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=info ts=2026-05-09T02:52:44.444416758Z caller=tables_manager.go:148 msg="compactor started" ts=2026-05-09T02:52:45.336932655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:41082" ts=2026-05-09T02:53:11.253741344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T02:53:23.641328276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T02:53:39.184883411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:44468" ts=2026-05-09T02:53:41.255398632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T02:54:11.256633463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T02:54:20.639428399Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T02:54:20.641923914Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T02:54:41.25828164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T02:54:45.342567377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:43776" level=debug ts=2026-05-09T02:54:55.571019873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T02:55:11.259762012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T02:55:31.897187797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T02:55:41.26090915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T02:55:45.34804791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:40112" level=debug ts=2026-05-09T02:56:09.86708025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T02:56:11.262070584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T02:56:41.263247515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T02:56:45.351602287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:44418" level=debug ts=2026-05-09T02:56:49.652437282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T02:57:09.194453905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:35308" ts=2026-05-09T02:57:11.265116012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T02:57:15.353400854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:39572" ts=2026-05-09T02:57:28.837125016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:53524" level=debug ts=2026-05-09T02:57:40.10153977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T02:57:40.10434786Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T02:57:41.266734322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T02:57:58.838097609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:36318" ts=2026-05-09T02:58:11.268019284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T02:58:29.995256683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T02:58:41.269653514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T02:58:58.840387333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:51654" ts=2026-05-09T02:59:11.270870468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T02:59:32.441779881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T02:59:39.201651685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:59704" ts=2026-05-09T02:59:41.272171268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T02:59:45.360523931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:48500" ts=2026-05-09T02:59:58.842883085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:46800" level=debug ts=2026-05-09T03:00:08.674552954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:00:09.203145382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:59782" ts=2026-05-09T03:00:11.273241769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:00:41.274815522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:00:58.847970453Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:58778" level=debug ts=2026-05-09T03:01:00.402641258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:01:11.276719557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:01:41.278222012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:01:58.85082271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:55094" level=debug ts=2026-05-09T03:01:58.965560129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T03:01:58.968291709Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:02:11.280295455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:02:41.281275532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=error ts=2026-05-09T03:02:44.449819735Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:02:44.451930819Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:02:44.454947149Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:02:44.454982126Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=debug ts=2026-05-09T03:02:48.880365034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:03:09.211404809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:41474" ts=2026-05-09T03:03:11.282972855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:03:41.284556561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:03:50.25581698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:04:11.285626187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:04:28.857815983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:59676" ts=2026-05-09T03:04:41.287230893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:04:48.746537247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:04:58.859792326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:59778" ts=2026-05-09T03:05:11.289096319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:05:15.378587172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:36580" ts=2026-05-09T03:05:41.290583251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:05:44.989415941Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:06:11.292101074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:06:28.878851948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:37504" ts=2026-05-09T03:06:41.293571007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T03:06:43.817539979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T03:06:43.819347103Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:06:58.880219032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:33352" ts=2026-05-09T03:07:11.296161631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:07:24.011487843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:07:28.881927416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:41556" ts=2026-05-09T03:07:39.22396297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:42384" ts=2026-05-09T03:07:41.297285045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:07:58.883256847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:47772" ts=2026-05-09T03:08:09.225618345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:56484" ts=2026-05-09T03:08:11.298882759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T03:08:16.263607048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:08:41.300424949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T03:09:04.811155676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:09:09.229098173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:48302" ts=2026-05-09T03:09:11.301994465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:09:41.303317928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:09:45.391449173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:58618" level=debug ts=2026-05-09T03:09:47.795242998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:09:58.888568546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:50154" ts=2026-05-09T03:10:09.232105069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:41704" ts=2026-05-09T03:10:11.305239277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:10:29.470820702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T03:10:29.472417149Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:10:41.306495402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:10:45.394584528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:49884" ts=2026-05-09T03:10:58.891374571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:52300" level=debug ts=2026-05-09T03:11:05.309459107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:11:11.308328574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:11:41.309222776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:11:45.396763895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:58344" ts=2026-05-09T03:11:58.894373776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:53282" level=debug ts=2026-05-09T03:11:59.332129218Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:12:11.310137999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:12:41.311375856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=error ts=2026-05-09T03:12:44.448617221Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:12:44.450349469Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:12:44.452026574Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:12:44.452047866Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=debug ts=2026-05-09T03:12:59.078480054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:13:11.312367339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:13:41.313351303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T03:13:53.776270305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:14:11.314397855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:14:15.402813811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:53714" level=debug ts=2026-05-09T03:14:32.604055164Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T03:14:32.605751763Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:14:39.244471246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:39466" ts=2026-05-09T03:14:41.316055307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:14:45.403932299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:36428" ts=2026-05-09T03:15:11.317832653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:15:19.764857753Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:15:41.318767462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:16:11.319773591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:16:13.235993236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:16:15.40829249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:53092" ts=2026-05-09T03:16:41.321209166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:17:04.272895929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:17:11.322416383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:17:39.253261416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:48714" ts=2026-05-09T03:17:41.323634324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:17:45.411995274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:35966" level=debug ts=2026-05-09T03:17:56.656424207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:17:58.911144243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:59166" ts=2026-05-09T03:18:09.254847485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:51032" ts=2026-05-09T03:18:11.325211743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:18:41.326845649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:18:45.41519718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:37636" level=debug ts=2026-05-09T03:18:52.984977046Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T03:18:52.986741745Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:18:58.913572275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:54654" ts=2026-05-09T03:19:11.327820137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:19:41.329372008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T03:19:48.816513424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:20:09.260487586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:35382" ts=2026-05-09T03:20:11.330456077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:20:28.918315178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:47486" level=debug ts=2026-05-09T03:20:38.606801171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:20:39.262335381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57702" ts=2026-05-09T03:20:41.332147768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:20:45.420046394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:47058" ts=2026-05-09T03:20:58.919350179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:42002" ts=2026-05-09T03:21:09.26385063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:46408" ts=2026-05-09T03:21:11.333060978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:21:15.421500098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:55444" ts=2026-05-09T03:21:28.921225757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:41492" level=debug ts=2026-05-09T03:21:32.781382438Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:21:39.265272861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:46530" ts=2026-05-09T03:21:41.334782127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:21:45.422528696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:37104" ts=2026-05-09T03:21:58.922760456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:51358" ts=2026-05-09T03:22:09.266891009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:54328" ts=2026-05-09T03:22:11.336364765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:22:21.47790089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:22:41.337762751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=error ts=2026-05-09T03:22:44.448935398Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:22:44.450754648Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:22:44.453224789Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:22:44.453245208Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:22:45.425258753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:44570" ts=2026-05-09T03:22:58.925694724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:58246" level=debug ts=2026-05-09T03:22:59.218571003Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T03:22:59.22030514Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:23:11.339369087Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:23:15.42670741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:34292" ts=2026-05-09T03:23:39.271308747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:36964" ts=2026-05-09T03:23:41.341015388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:23:49.47491094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:24:11.341901775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T03:24:30.789495813Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:24:41.343535177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:25:11.344373556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T03:25:12.088938923Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:25:39.276620636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:33436" ts=2026-05-09T03:25:41.346183957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:25:45.432988889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:50706" level=debug ts=2026-05-09T03:26:03.572819745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:26:11.347423247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:26:15.434440915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:44664" level=debug ts=2026-05-09T03:26:38.181762921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T03:26:38.183231711Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:26:39.279342429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:45348" ts=2026-05-09T03:26:41.348554562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:26:45.436028322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:53888" ts=2026-05-09T03:26:58.937485257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:51646" ts=2026-05-09T03:27:11.349853617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:27:36.715120719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:27:41.351877453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:28:11.353957014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:28:19.356945634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:28:41.355391539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:28:45.441127529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:37834" level=debug ts=2026-05-09T03:28:58.06211576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:29:09.285589934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:52622" ts=2026-05-09T03:29:11.3570017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:29:15.442919629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:39858" ts=2026-05-09T03:29:41.358944104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T03:29:56.704539999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:30:11.360360063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T03:30:32.868757874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T03:30:32.870498451Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:30:39.289111536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:43960" ts=2026-05-09T03:30:41.361415772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:31:09.170896802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:31:09.290065755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:50622" ts=2026-05-09T03:31:11.362305825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:31:15.448754427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:34714" ts=2026-05-09T03:31:28.953915286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:43050" ts=2026-05-09T03:31:39.291470808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:47032" ts=2026-05-09T03:31:41.363637681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T03:31:55.528291414Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:32:11.36519309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:32:28.957086591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:40132" ts=2026-05-09T03:32:41.366895604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=error ts=2026-05-09T03:32:44.448308127Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:32:44.450289826Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:32:44.45265086Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:32:44.452689093Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:32:45.453372441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:42298" level=debug ts=2026-05-09T03:32:50.608253705Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:33:11.3678807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:33:38.518386792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:33:41.369148359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:33:58.961154461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:40988" ts=2026-05-09T03:34:11.370728684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:34:15.45780024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:58984" level=debug ts=2026-05-09T03:34:27.300995223Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T03:34:27.302635446Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:34:28.962991882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:38666" ts=2026-05-09T03:34:39.29937425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:50668" ts=2026-05-09T03:34:41.372020991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:34:45.458940723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:51770" ts=2026-05-09T03:34:58.964266012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:55738" level=debug ts=2026-05-09T03:35:10.085720487Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:35:11.372981257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:35:15.460029717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:33976" ts=2026-05-09T03:35:41.37440436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T03:35:47.016404183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:36:11.376048003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T03:36:30.062718643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:36:41.376982245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:36:58.969500008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:53534" level=debug ts=2026-05-09T03:37:07.953446996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:37:11.378129901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:37:15.465523647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:57888" ts=2026-05-09T03:37:39.307137581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:50318" ts=2026-05-09T03:37:41.379786408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:37:45.466886154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:58396" ts=2026-05-09T03:37:58.971883007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:49278" level=debug ts=2026-05-09T03:38:07.362817466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T03:38:07.364588219Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:38:11.381384354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:38:28.973139161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:35980" ts=2026-05-09T03:38:41.383147109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T03:39:09.047698961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:39:11.38480608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:39:15.470398075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:38582" ts=2026-05-09T03:39:41.386032854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:39:45.472207868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:36336" level=debug ts=2026-05-09T03:40:00.780152023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:40:09.31481803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57068" ts=2026-05-09T03:40:11.387211595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:40:15.473137132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:39728" ts=2026-05-09T03:40:28.979396553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:36668" ts=2026-05-09T03:40:41.388269114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:40:45.474801378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:43390" level=debug ts=2026-05-09T03:40:58.003091382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:41:09.317206687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57800" ts=2026-05-09T03:41:11.389306957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:41:41.390837779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:41:47.684409465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:41:58.984849472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:50612" ts=2026-05-09T03:42:11.392402815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:42:41.394048462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=error ts=2026-05-09T03:42:44.447434241Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:42:44.449198318Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:42:44.450953219Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:42:44.450981183Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=debug ts=2026-05-09T03:42:45.80079905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T03:42:45.802373677Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:42:58.98725437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:36096" ts=2026-05-09T03:43:11.395486307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T03:43:32.393282865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:43:41.39641056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:44:09.324262449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:45940" ts=2026-05-09T03:44:11.398075378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:44:15.485460139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:37662" level=debug ts=2026-05-09T03:44:29.494060461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:44:41.39985338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:44:45.486369106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:45868" level=debug ts=2026-05-09T03:45:08.88775429Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:45:11.401307846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:45:41.402886817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:46:08.930721325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:46:11.403957173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:46:41.404906632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T03:47:00.970268334Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T03:47:00.971807278Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:47:11.406026544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:47:15.493559324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:34658" ts=2026-05-09T03:47:39.333418211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:59272" ts=2026-05-09T03:47:41.407534512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:47:57.98308865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:47:58.998696191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:45234" ts=2026-05-09T03:48:11.408530494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:48:28.999566144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:36002" level=debug ts=2026-05-09T03:48:35.417025616Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:48:39.335912603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57090" ts=2026-05-09T03:48:41.410086988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:49:11.411324341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:49:13.646483883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:49:39.339386095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57826" ts=2026-05-09T03:49:41.412739424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:49:52.968532004Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:50:11.414387218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T03:50:27.777611213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T03:50:27.779382682Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:50:41.415254822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:51:02.86331624Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:51:11.416692043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:51:39.344603098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:38898" ts=2026-05-09T03:51:41.417995696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T03:51:59.461708537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:52:09.345615955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:35730" ts=2026-05-09T03:52:11.419366696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:52:41.420431748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:52:42.445623186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T03:52:44.447543468Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:52:44.449111775Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:52:44.450619713Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T03:52:44.45064126Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:53:11.421583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:53:29.010942505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:59176" level=debug ts=2026-05-09T03:53:33.381895469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:53:39.348753115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:60882" ts=2026-05-09T03:53:41.42268414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:54:11.42406009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:54:12.518949599Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T03:54:12.520464352Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:54:29.0125771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:52424" ts=2026-05-09T03:54:41.424943668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:54:45.515211215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:42564" level=debug ts=2026-05-09T03:55:04.903052976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:55:09.35199003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:36830" ts=2026-05-09T03:55:11.426512276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:55:15.516198209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:51372" ts=2026-05-09T03:55:29.015709584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:32868" level=debug ts=2026-05-09T03:55:41.264269693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:55:41.427932817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T03:56:11.429265697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T03:56:18.688080245Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:56:41.430237132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T03:56:56.251818731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:57:11.431157335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:57:15.520920034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:33102" level=debug ts=2026-05-09T03:57:36.229467647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T03:57:36.23099465Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T03:57:41.432229038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T03:57:59.021961055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:55112" ts=2026-05-09T03:58:09.360462925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:36656" ts=2026-05-09T03:58:11.433963801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:58:19.701936134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:58:41.435177462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T03:59:09.36331126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:51434" ts=2026-05-09T03:59:11.436589165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T03:59:16.175100413Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T03:59:41.43749444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T03:59:52.477042665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:00:11.439307125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T04:00:41.440718387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T04:00:42.336010894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:00:59.030101652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:41338" ts=2026-05-09T04:01:09.368583679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:51702" ts=2026-05-09T04:01:11.441717181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T04:01:20.587951975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T04:01:20.589603854Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T04:01:29.031403104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:34778" ts=2026-05-09T04:01:39.370379529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:33818" ts=2026-05-09T04:01:41.442719985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T04:01:45.533480002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:57150" level=debug ts=2026-05-09T04:01:55.418815382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:01:59.033266542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:45848" ts=2026-05-09T04:02:09.371908945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:40028" ts=2026-05-09T04:02:11.444391221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T04:02:35.288501872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:02:39.373595565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57730" ts=2026-05-09T04:02:41.445928884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=error ts=2026-05-09T04:02:44.447596483Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T04:02:44.449223048Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T04:02:44.450637972Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T04:02:44.450671406Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T04:03:09.375009884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:40272" level=debug ts=2026-05-09T04:03:10.044990414Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:03:11.447115681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T04:03:29.036279827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:39628" ts=2026-05-09T04:03:41.448611181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T04:04:09.377723701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:51812" ts=2026-05-09T04:04:11.450082949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T04:04:12.132291804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:04:41.451567574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T04:04:59.504320692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T04:04:59.505992993Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T04:05:11.453183129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T04:05:41.45423035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T04:05:45.54408757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:53392" level=debug ts=2026-05-09T04:05:47.070065437Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:06:11.455383972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T04:06:15.545107242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:54794" level=debug ts=2026-05-09T04:06:21.828837093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:06:39.384323516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:48886" ts=2026-05-09T04:06:41.456913421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T04:07:00.385740258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:07:11.458357859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T04:07:39.386909033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:34096" ts=2026-05-09T04:07:41.459362868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" level=debug ts=2026-05-09T04:07:53.642799056Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:08:09.388638722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:53200" ts=2026-05-09T04:08:11.460715133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T04:08:15.550219152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:45002" level=debug ts=2026-05-09T04:08:34.729939989Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T04:08:34.731595495Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T04:08:41.461568572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T04:09:11.462514392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T04:09:24.642788284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:09:41.464000029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T04:10:10.609000873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:10:11.465547861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T04:10:15.555544006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:52612" ts=2026-05-09T04:10:41.467247251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T04:10:45.556755132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:60938" level=debug ts=2026-05-09T04:11:11.29481698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:11:11.468582335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T04:11:29.060690632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:40152" ts=2026-05-09T04:11:41.470177001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T04:11:45.559384563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:40970" ts=2026-05-09T04:12:11.471736931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T04:12:13.154060931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:12:41.472756244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=error ts=2026-05-09T04:12:44.448649942Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T04:12:44.450453501Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T04:12:44.452037833Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-09T04:12:44.45206711Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=debug ts=2026-05-09T04:12:48.322714591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T04:12:48.324211451Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T04:13:11.473727542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T04:13:23.495942284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:13:41.47497214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T04:13:45.564545909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:56142" ts=2026-05-09T04:13:59.068473949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:53028" ts=2026-05-09T04:14:11.476209002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T04:14:11.560781653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:14:41.477253495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" level=debug ts=2026-05-09T04:14:49.687632892Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:15:09.406587978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:51698" ts=2026-05-09T04:15:11.478686898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T04:15:29.160002938Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:15:41.480125808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T04:15:45.569618554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:57982" ts=2026-05-09T04:16:11.481315188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T04:16:19.636430832Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-09T04:16:19.637907422Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-09T04:16:41.482625251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-qgdkq-8568266f 10.128.0.77:7946" ts=2026-05-09T04:17:09.412689248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:46042" level=debug ts=2026-05-09T04:17:10.986254542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:17:11.483697717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T04:17:41.48504588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" ts=2026-05-09T04:17:45.575821493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:33160" level=debug ts=2026-05-09T04:18:02.708303558Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:18:11.486127287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-96870eab 10.128.0.82:7946" ts=2026-05-09T04:18:15.576778103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:48444" ts=2026-05-09T04:18:41.487848107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946" level=debug ts=2026-05-09T04:19:02.248908417Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-09T04:19:11.48865058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nq86b-1c2d7a36 10.128.0.76:7946"