level=info ts=2026-04-27T22:44:36.947527895Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)" level=info ts=2026-04-27T22:44:36.947566435Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml level=info ts=2026-04-27T22:44:36.949412636Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095 level=debug ts=2026-04-27T22:44:36.949798597Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-04-27T22:44:36.949905858Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.84 inf=eth0 level=debug ts=2026-04-27T22:44:36.95037887Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-04-27T22:44:36.950462208Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.84 inf=eth0 level=info ts=2026-04-27T22:44:36.950477684Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-4bff1c73 ts=2026-04-27T22:44:36.950556444Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired" level=debug ts=2026-04-27T22:44:36.950572762Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.84 advertisePort=7946 level=debug ts=2026-04-27T22:44:36.950594148Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.84 advertisePort=7946 level=info ts=2026-04-27T22:44:36.950751748Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4 ts=2026-04-27T22:44:36.956552455Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=info ts=2026-04-27T22:44:36.956566933Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" level=warn ts=2026-04-27T22:44:36.956577891Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.828211ms level=error ts=2026-04-27T22:44:36.956588346Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure" level=info ts=2026-04-27T22:44:36.956603075Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local ts=2026-04-27T22:44:36.96080358Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-04-27T22:44:36.960822029Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" level=info ts=2026-04-27T22:44:36.96577374Z caller=module_service.go:82 msg=starting module=analytics level=info ts=2026-04-27T22:44:36.965798043Z caller=module_service.go:82 msg=starting module=runtime-config level=info ts=2026-04-27T22:44:36.965789351Z caller=module_service.go:82 msg=starting module=server level=debug ts=2026-04-27T22:44:36.965824991Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server level=info ts=2026-04-27T22:44:36.96595034Z caller=module_service.go:82 msg=starting module=memberlist-kv level=debug ts=2026-04-27T22:44:36.965981277Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics level=debug ts=2026-04-27T22:44:36.966158432Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv level=debug ts=2026-04-27T22:44:36.966189242Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config level=debug ts=2026-04-27T22:44:36.966201229Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server level=info ts=2026-04-27T22:44:36.966204655Z caller=module_service.go:82 msg=starting module=compactor level=info ts=2026-04-27T22:44:36.966274143Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor level=info ts=2026-04-27T22:44:36.966299695Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty" level=info ts=2026-04-27T22:44:36.966371276Z caller=ring.go:365 msg="ring doesn't exist in KV store yet" level=info ts=2026-04-27T22:44:36.966488141Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring" level=info ts=2026-04-27T22:44:37.156862087Z caller=compactor.go:387 msg="compactor is JOINING in the ring" level=info ts=2026-04-27T22:44:37.156954561Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring" level=info ts=2026-04-27T22:44:37.337460038Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring" level=info ts=2026-04-27T22:44:37.337516097Z caller=loki.go:599 msg="Loki started" startup_time=459.402802ms ts=2026-04-27T22:44:38.219479372Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-04-27T22:44:38.219507017Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" ts=2026-04-27T22:44:41.801876823Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-04-27T22:44:41.80191652Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" level=info ts=2026-04-27T22:44:42.338385938Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor" level=info ts=2026-04-27T22:44:42.338474253Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor" ts=2026-04-27T22:44:49.151577866Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-04-27T22:44:49.151604829Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" level=debug ts=2026-04-27T22:44:53.783165038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-27T22:44:54.235485064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37836" ts=2026-04-27T22:44:59.409502995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:58678" ts=2026-04-27T22:45:02.709821526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:42572" ts=2026-04-27T22:45:03.994902375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.128.0.84:7946" ts=2026-04-27T22:45:03.995034026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:34260" ts=2026-04-27T22:45:03.996442606Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.82:7946: dial tcp 10.128.0.82:7946: connect: connection refused" ts=2026-04-27T22:45:03.996585401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.128.0.83:7946" ts=2026-04-27T22:45:03.997105213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.128.0.81:7946" level=info ts=2026-04-27T22:45:03.9975427Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=27.040933799s ts=2026-04-27T22:45:14.162881001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:43678" level=debug ts=2026-04-27T22:45:14.978040151Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-27T22:45:20.839233204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-08e6a14e 10.128.0.85:7946" level=debug ts=2026-04-27T22:45:37.748119939Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-27T22:45:44.164816366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56454" ts=2026-04-27T22:45:50.84015656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T22:46:03.816294252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-27T22:46:20.841922313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T22:46:24.600998986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=error ts=2026-04-27T22:46:24.602639401Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T22:46:50.84349058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T22:47:03.967896745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-27T22:47:20.845211974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T22:47:35.786942361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:46812" ts=2026-04-27T22:47:50.846074686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T22:48:01.357152839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-27T22:48:14.172020576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:33748" ts=2026-04-27T22:48:20.847285788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T22:48:35.789303717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:49732" level=debug ts=2026-04-27T22:48:49.855923802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-27T22:48:50.849130152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-08e6a14e 10.128.0.85:7946" ts=2026-04-27T22:49:05.790392769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:35142" ts=2026-04-27T22:49:20.850809628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T22:49:29.599697132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:48152" ts=2026-04-27T22:49:50.852336857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T22:50:02.0171684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-27T22:50:05.793416029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:48690" ts=2026-04-27T22:50:20.853936666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T22:50:35.794831154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:50538" ts=2026-04-27T22:50:50.854955834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" level=debug ts=2026-04-27T22:50:58.837013848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=error ts=2026-04-27T22:50:58.838726262Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T22:51:05.795987752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:41094" ts=2026-04-27T22:51:20.856011192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-08e6a14e 10.128.0.85:7946" level=debug ts=2026-04-27T22:51:44.144897331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T22:51:44.182629533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:54172" ts=2026-04-27T22:51:50.857003205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T22:51:59.606679709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:34774" ts=2026-04-27T22:52:20.858759086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" level=debug ts=2026-04-27T22:52:22.90253059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T22:52:35.799780232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:59148" ts=2026-04-27T22:52:44.185610357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:47774" ts=2026-04-27T22:52:50.85999754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-08e6a14e 10.128.0.85:7946" level=debug ts=2026-04-27T22:53:11.964671584Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T22:53:20.861401386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T22:53:44.188807584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:41680" ts=2026-04-27T22:53:50.862784236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T22:53:54.441723149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T22:53:59.612702805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:57128" ts=2026-04-27T22:54:20.864240082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" level=debug ts=2026-04-27T22:54:35.79225956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T22:54:35.79423999Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T22:54:35.804276906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:47432" level=info ts=2026-04-27T22:54:42.338892513Z caller=tables_manager.go:76 msg="compactor startup delay completed" level=error ts=2026-04-27T22:54:42.342764223Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T22:54:42.344570711Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T22:54:42.346229525Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T22:54:42.346269644Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=info ts=2026-04-27T22:54:42.346283338Z caller=tables_manager.go:145 msg="compactor started" ts=2026-04-27T22:54:44.191836138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:39854" ts=2026-04-27T22:54:50.865977474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" level=debug ts=2026-04-27T22:55:20.576258025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T22:55:20.867303636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T22:55:35.806800046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:42964" ts=2026-04-27T22:55:50.868492798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T22:56:05.807861254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:53904" level=debug ts=2026-04-27T22:56:07.054690197Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T22:56:20.869461412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T22:56:35.809255545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:59474" level=debug ts=2026-04-27T22:56:41.69790489Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T22:56:44.19682891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:60744" ts=2026-04-27T22:56:50.870676296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T22:57:14.197788194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:36944" ts=2026-04-27T22:57:20.872388501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" level=debug ts=2026-04-27T22:57:24.877723706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T22:57:50.873426214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" level=debug ts=2026-04-27T22:58:13.368459617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T22:58:13.370302665Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T22:58:20.875032358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T22:58:35.814654082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:55878" level=debug ts=2026-04-27T22:58:49.083503803Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T22:58:50.876678407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T22:59:20.877893086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-08e6a14e 10.128.0.85:7946" level=debug ts=2026-04-27T22:59:40.208631056Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T22:59:44.205934378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:51628" ts=2026-04-27T22:59:50.879014366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" level=debug ts=2026-04-27T23:00:19.429187858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:00:20.880210895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-08e6a14e 10.128.0.85:7946" ts=2026-04-27T23:00:50.881890833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:01:05.993803811Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:01:20.883339065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:01:44.212336392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:51306" ts=2026-04-27T23:01:50.884659103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" level=debug ts=2026-04-27T23:01:51.995896464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:01:51.99763734Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:01:59.636271214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:47694" ts=2026-04-27T23:02:20.886372994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-08e6a14e 10.128.0.85:7946" level=debug ts=2026-04-27T23:02:32.245131602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:02:50.888123869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:02:59.640643235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:50578" level=debug ts=2026-04-27T23:03:09.446635798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:03:14.216799931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:58100" ts=2026-04-27T23:03:20.889857068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:03:50.890757122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:03:55.470212004Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:04:05.831937231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:52604" ts=2026-04-27T23:04:14.219270051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56538" ts=2026-04-27T23:04:20.891785644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=error ts=2026-04-27T23:04:42.352402137Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:04:42.355462724Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:04:42.358288602Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:04:42.358338867Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:04:44.221072816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:48992" level=debug ts=2026-04-27T23:04:45.263529611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:04:50.893342759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:05:20.894267668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-08e6a14e 10.128.0.85:7946" level=debug ts=2026-04-27T23:05:29.265764979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:05:29.267680207Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:05:50.895187504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" level=debug ts=2026-04-27T23:06:06.6330686Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:06:14.22580293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:47104" ts=2026-04-27T23:06:20.896630083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-08e6a14e 10.128.0.85:7946" ts=2026-04-27T23:06:50.89782118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-08e6a14e 10.128.0.85:7946" ts=2026-04-27T23:06:59.65288642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:44202" level=debug ts=2026-04-27T23:07:07.937165014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:07:20.89907492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-08e6a14e 10.128.0.85:7946" ts=2026-04-27T23:07:50.900877078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:07:51.188777848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:08:20.902461432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-08e6a14e 10.128.0.85:7946" ts=2026-04-27T23:08:44.23238348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:41814" level=debug ts=2026-04-27T23:08:44.868639399Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:08:50.903727094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:09:05.858246057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:47154" ts=2026-04-27T23:09:14.233960853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:52058" ts=2026-04-27T23:09:20.905708723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:09:31.210129845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:09:31.21180364Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:09:50.906722043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-08e6a14e 10.128.0.85:7946" ts=2026-04-27T23:10:20.907852672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" level=debug ts=2026-04-27T23:10:24.898847539Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:10:44.237664591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53614" ts=2026-04-27T23:10:50.909702815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:10:59.681911364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:42262" ts=2026-04-27T23:11:20.911124078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" level=debug ts=2026-04-27T23:11:21.244862019Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:11:50.912604458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-08e6a14e 10.128.0.85:7946" ts=2026-04-27T23:11:59.68465706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38198" level=debug ts=2026-04-27T23:12:20.635138306Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:12:20.91417443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:12:35.86808845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:52154" ts=2026-04-27T23:12:43.026922396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:54398" ts=2026-04-27T23:12:43.036544647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:54404" ts=2026-04-27T23:12:48.952472112Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-08e6a14e (timeout reached)" ts=2026-04-27T23:12:50.918542275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:12:51.952521749Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-08e6a14e has failed, no acks received" ts=2026-04-27T23:12:58.952397129Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-08e6a14e (timeout reached)" ts=2026-04-27T23:13:01.952254793Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-08e6a14e has failed, no acks received" ts=2026-04-27T23:13:11.953600566Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-08e6a14e as failed, suspect timeout reached (2 peer confirmations)" level=debug ts=2026-04-27T23:13:18.277612152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:13:20.91977532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:13:35.870628395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:52630" ts=2026-04-27T23:13:50.920794955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-298577a2 10.128.0.85:7946" level=debug ts=2026-04-27T23:13:53.169334565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:13:53.171461376Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:13:59.690628844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:50124" ts=2026-04-27T23:14:20.922465578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:14:39.711487042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:14:42.34879872Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:14:42.350914194Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:14:42.352605374Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:14:42.35263461Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:14:50.924204965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-298577a2 10.128.0.85:7946" ts=2026-04-27T23:15:05.874802587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:42754" ts=2026-04-27T23:15:20.925795304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-298577a2 10.128.0.85:7946" level=debug ts=2026-04-27T23:15:23.944185751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:15:40.021063504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:59778" ts=2026-04-27T23:15:50.926921981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:16:18.259173754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:16:20.928329939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:16:35.878803464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:58332" ts=2026-04-27T23:16:50.929696583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:17:05.623821786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:17:20.930496952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:17:35.881886878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:37782" ts=2026-04-27T23:17:50.931544979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" level=debug ts=2026-04-27T23:17:55.047760247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:17:55.049786023Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:18:20.932734264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:18:47.464597642Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:18:50.933879468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:19:20.934797717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:19:29.704472193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37564" level=debug ts=2026-04-27T23:19:31.246586167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:19:35.886747308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:37570" ts=2026-04-27T23:19:40.031483131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:39572" ts=2026-04-27T23:19:50.935652336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:19:59.705688783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:49302" ts=2026-04-27T23:20:20.936960913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-298577a2 10.128.0.85:7946" level=debug ts=2026-04-27T23:20:32.073000725Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:20:33.951611541Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-298577a2 (timeout reached)" ts=2026-04-27T23:20:35.88955593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:54198" ts=2026-04-27T23:20:36.951391063Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-298577a2 has failed, no acks received" ts=2026-04-27T23:20:43.951735052Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-298577a2 (timeout reached)" ts=2026-04-27T23:20:45.3813927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:50692" ts=2026-04-27T23:20:45.392323012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:50704" ts=2026-04-27T23:20:46.951375718Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-298577a2 has failed, no acks received" ts=2026-04-27T23:20:50.938169656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6c499f1e 10.128.0.85:7946" ts=2026-04-27T23:20:56.935823773Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-298577a2 as failed, suspect timeout reached (2 peer confirmations)" ts=2026-04-27T23:21:05.89121756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:50060" ts=2026-04-27T23:21:20.939821716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6c499f1e 10.128.0.85:7946" level=debug ts=2026-04-27T23:21:34.279954811Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:21:50.940908557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:22:20.942395826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:22:26.763806541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:22:26.765561028Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:22:34.362925172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56848" ts=2026-04-27T23:22:50.943629776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:23:01.877229103Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:23:04.363718606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:40968" ts=2026-04-27T23:23:05.896975887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:51132" ts=2026-04-27T23:23:20.944551277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6c499f1e 10.128.0.85:7946" ts=2026-04-27T23:23:29.714910062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36772" level=debug ts=2026-04-27T23:23:44.549489927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:23:50.945820875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6c499f1e 10.128.0.85:7946" ts=2026-04-27T23:24:04.366384173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:39910" ts=2026-04-27T23:24:05.90005349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:40766" ts=2026-04-27T23:24:20.946963193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:24:29.717243995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:58176" level=debug ts=2026-04-27T23:24:34.874143979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:24:42.349381113Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:24:42.351986063Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:24:42.353607981Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:24:42.353640374Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:24:50.948242689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6c499f1e 10.128.0.85:7946" ts=2026-04-27T23:25:20.950743177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:25:28.462802438Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:25:50.95285558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:26:20.953897208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6c499f1e 10.128.0.85:7946" level=debug ts=2026-04-27T23:26:23.646677071Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:26:23.648526532Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:26:34.372429508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56308" ts=2026-04-27T23:26:50.954924934Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-6c499f1e failed: dial tcp 10.128.0.85:7946: connect: connection refused" ts=2026-04-27T23:26:53.951823647Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-6c499f1e (timeout reached)" ts=2026-04-27T23:26:56.952116358Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-6c499f1e has failed, no acks received" ts=2026-04-27T23:27:03.952372194Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-6c499f1e (timeout reached)" ts=2026-04-27T23:27:05.907107955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:40590" ts=2026-04-27T23:27:06.952127746Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-6c499f1e has failed, no acks received" ts=2026-04-27T23:27:08.541089079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:51416" ts=2026-04-27T23:27:08.550110432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:51426" ts=2026-04-27T23:27:16.933141928Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-6c499f1e as failed, suspect timeout reached (2 peer confirmations)" level=debug ts=2026-04-27T23:27:17.375550907Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:27:20.955863853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:27:39.028979528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56508" ts=2026-04-27T23:27:50.957548831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-97e8ee35 10.128.0.85:7946" level=debug ts=2026-04-27T23:28:15.29581653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:28:20.958391714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:28:29.728339908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:42576" ts=2026-04-27T23:28:50.959531542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:29:05.911706811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:47282" ts=2026-04-27T23:29:09.032821114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:49078" level=debug ts=2026-04-27T23:29:15.398336961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:29:20.96033061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-97e8ee35 10.128.0.85:7946" ts=2026-04-27T23:29:35.912860484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:55002" level=debug ts=2026-04-27T23:29:49.310912827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:29:50.961686728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-97e8ee35 10.128.0.85:7946" ts=2026-04-27T23:30:05.913969962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:54440" ts=2026-04-27T23:30:20.962587251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:30:39.036732971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:34476" level=debug ts=2026-04-27T23:30:44.438872985Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:30:44.44056838Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:30:50.96415817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:31:20.966070137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-97e8ee35 10.128.0.85:7946" ts=2026-04-27T23:31:29.735024097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:59980" level=debug ts=2026-04-27T23:31:32.399176528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:31:39.039079189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56730" ts=2026-04-27T23:31:50.971755595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-97e8ee35 10.128.0.85:7946" ts=2026-04-27T23:31:59.736303647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:42140" ts=2026-04-27T23:32:09.040117323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:33766" ts=2026-04-27T23:32:20.973871731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:32:21.401440921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:32:29.737344487Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:47820" ts=2026-04-27T23:32:39.041174516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:49768" ts=2026-04-27T23:32:50.974770228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:32:57.555098465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:33:05.921369733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:42978" ts=2026-04-27T23:33:20.975803801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:33:47.104010661Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:33:50.977439772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:34:09.045721894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:39810" ts=2026-04-27T23:34:20.978655376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:34:29.74241098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:60916" level=debug ts=2026-04-27T23:34:32.065661535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:34:32.06742061Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:34:42.350010461Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:34:42.352040173Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:34:42.353919716Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:34:42.353946991Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:34:50.980349503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:35:19.506685339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:35:20.981797909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:35:43.951804071Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-97e8ee35 (timeout reached)" ts=2026-04-27T23:35:46.951424302Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-97e8ee35 has failed, no acks received" ts=2026-04-27T23:35:50.983470302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:35:59.746125854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:44356" ts=2026-04-27T23:36:03.952477826Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-97e8ee35 (timeout reached)" ts=2026-04-27T23:36:05.931421694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:60528" ts=2026-04-27T23:36:06.952287178Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-97e8ee35 has failed, no acks received" ts=2026-04-27T23:36:06.952392527Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-97e8ee35 as failed, suspect timeout reached (2 peer confirmations)" level=debug ts=2026-04-27T23:36:18.452887591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:36:20.984513337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:36:25.232406037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:51198" ts=2026-04-27T23:36:25.241996489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:51212" ts=2026-04-27T23:36:35.932580109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:57884" ts=2026-04-27T23:36:50.985582269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-effdddb8 10.128.0.85:7946" level=debug ts=2026-04-27T23:37:11.411480742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:37:20.98701917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-effdddb8 10.128.0.85:7946" ts=2026-04-27T23:37:50.988520828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:37:51.870983965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:38:20.989760107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-effdddb8 10.128.0.85:7946" level=debug ts=2026-04-27T23:38:46.073075931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:38:46.074681696Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:38:50.990955793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:39:00.217130061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53578" ts=2026-04-27T23:39:20.992692379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" level=debug ts=2026-04-27T23:39:34.542043665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:39:50.993670662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:40:00.22028056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:32984" ts=2026-04-27T23:40:20.994619258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:40:29.75961194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:42210" ts=2026-04-27T23:40:30.221191403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:41924" level=debug ts=2026-04-27T23:40:35.453902307Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:40:50.995841136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:40:59.760650251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:59090" level=debug ts=2026-04-27T23:41:19.032425429Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:41:20.997004287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:41:29.762505017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:56728" ts=2026-04-27T23:41:50.998516124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-effdddb8 10.128.0.85:7946" ts=2026-04-27T23:42:05.946990421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:53976" level=debug ts=2026-04-27T23:42:20.783582936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:42:20.999396032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:42:29.76423955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:49534" ts=2026-04-27T23:42:43.951474238Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-effdddb8 (timeout reached)" ts=2026-04-27T23:42:46.951409893Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-effdddb8 has failed, no acks received" ts=2026-04-27T23:42:48.95175948Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-effdddb8 (timeout reached)" ts=2026-04-27T23:42:51.000711471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:42:51.95156048Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-effdddb8 has failed, no acks received" level=debug ts=2026-04-27T23:43:05.3300472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:43:05.332013756Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:43:21.002448546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:43:29.767187831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:45102" ts=2026-04-27T23:43:35.950949704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:33108" ts=2026-04-27T23:43:51.003678413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:43:51.241174861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:45626" ts=2026-04-27T23:43:51.251464487Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:45634" ts=2026-04-27T23:43:59.768231013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36272" level=debug ts=2026-04-27T23:44:02.667070673Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:44:05.952382949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:39650" ts=2026-04-27T23:44:21.004873796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76680a44 10.128.0.85:7946" ts=2026-04-27T23:44:32.746672551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:59578" ts=2026-04-27T23:44:35.954036152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:37762" level=debug ts=2026-04-27T23:44:39.92843364Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:44:42.349554735Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:44:42.351651878Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:44:42.353491911Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:44:42.353517922Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:44:51.006008808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76680a44 10.128.0.85:7946" ts=2026-04-27T23:45:21.007342855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76680a44 10.128.0.85:7946" ts=2026-04-27T23:45:29.771710221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:34418" level=debug ts=2026-04-27T23:45:35.097449504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:45:51.008819588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76680a44 10.128.0.85:7946" ts=2026-04-27T23:46:02.750551005Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:36852" ts=2026-04-27T23:46:21.010396885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76680a44 10.128.0.85:7946" ts=2026-04-27T23:46:32.751646043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:35712" level=debug ts=2026-04-27T23:46:36.034223731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:46:51.011203175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:47:21.012715112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76680a44 10.128.0.85:7946" ts=2026-04-27T23:47:29.77725609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:47606" level=debug ts=2026-04-27T23:47:31.887966133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:47:31.889573672Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:47:51.014272539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76680a44 10.128.0.85:7946" ts=2026-04-27T23:47:59.77858146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:47448" ts=2026-04-27T23:48:02.755116623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:40156" ts=2026-04-27T23:48:21.015954154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76680a44 10.128.0.85:7946" level=debug ts=2026-04-27T23:48:26.247240732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:48:51.017600965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:48:59.780822783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:33648" ts=2026-04-27T23:49:02.757733534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:34434" level=debug ts=2026-04-27T23:49:04.385861803Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:49:05.965030864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:37986" ts=2026-04-27T23:49:21.019096977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:49:35.96611503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:46658" ts=2026-04-27T23:49:51.020164477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:49:54.867887048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:50:21.021083607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:50:29.785644574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:34856" level=debug ts=2026-04-27T23:50:48.555968233Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:50:51.021872128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76680a44 10.128.0.85:7946" ts=2026-04-27T23:51:21.02347381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76680a44 10.128.0.85:7946" level=debug ts=2026-04-27T23:51:49.510951672Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:51:49.51244951Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:51:51.024451156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:52:02.765965026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:57454" ts=2026-04-27T23:52:21.025949537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:52:32.767476193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:52802" ts=2026-04-27T23:52:35.973147045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:59564" level=debug ts=2026-04-27T23:52:45.828259071Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:52:51.027663752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-76680a44 10.128.0.85:7946" ts=2026-04-27T23:52:59.793213749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:54816" ts=2026-04-27T23:53:02.769205615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:35522" ts=2026-04-27T23:53:05.974349503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:48066" ts=2026-04-27T23:53:21.028979057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:53:35.975495442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:52542" ts=2026-04-27T23:53:43.952457511Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-76680a44 (timeout reached)" ts=2026-04-27T23:53:46.933529307Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-76680a44 as failed, suspect timeout reached (2 peer confirmations)" level=debug ts=2026-04-27T23:53:46.948475815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:53:46.951749952Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-76680a44 has failed, no acks received" ts=2026-04-27T23:53:51.030183944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-27T23:54:01.219335691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:33092" ts=2026-04-27T23:54:01.228516866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:33102" ts=2026-04-27T23:54:05.97670365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:40944" ts=2026-04-27T23:54:21.031079497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" level=debug ts=2026-04-27T23:54:21.165060385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:54:42.349883978Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:54:42.351792741Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:54:42.353542012Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-27T23:54:42.353566879Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:54:51.032168844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8854ca5d 10.128.0.85:7946" level=debug ts=2026-04-27T23:55:00.063639902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:55:21.033612312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:55:51.034650476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8854ca5d 10.128.0.85:7946" level=debug ts=2026-04-27T23:55:55.870383318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-27T23:55:55.871975685Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-27T23:55:59.800554837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:58592" ts=2026-04-27T23:56:05.982240929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:42254" ts=2026-04-27T23:56:21.035622113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8854ca5d 10.128.0.85:7946" level=debug ts=2026-04-27T23:56:49.337317606Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:56:51.036979265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:57:12.180404561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:52566" ts=2026-04-27T23:57:21.037766043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:57:29.804903798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:40300" level=debug ts=2026-04-27T23:57:31.124288514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:57:35.986211735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:39558" ts=2026-04-27T23:57:51.039038405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:57:59.806088953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37838" ts=2026-04-27T23:58:05.987562785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:43610" ts=2026-04-27T23:58:21.040583136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-27T23:58:24.976172634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:58:29.807434643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:47404" ts=2026-04-27T23:58:35.988464581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:49468" ts=2026-04-27T23:58:51.041985307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-27T23:59:05.990039328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:34244" level=debug ts=2026-04-27T23:59:16.019159561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-27T23:59:21.043087195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8854ca5d 10.128.0.85:7946" ts=2026-04-27T23:59:35.991606959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:49322" ts=2026-04-27T23:59:51.044062344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" level=debug ts=2026-04-28T00:00:03.38016698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T00:00:03.382721698Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T00:00:21.045190362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-28T00:00:42.18879235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:40544" level=debug ts=2026-04-28T00:00:49.46617581Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:00:51.046601599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-28T00:01:21.04784745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-28T00:01:39.848692596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:01:51.048800914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8854ca5d 10.128.0.85:7946" ts=2026-04-28T00:01:58.951300846Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8854ca5d (timeout reached)" ts=2026-04-28T00:02:01.95129259Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8854ca5d has failed, no acks received" ts=2026-04-28T00:02:13.952029784Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8854ca5d (timeout reached)" ts=2026-04-28T00:02:16.952187071Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8854ca5d has failed, no acks received" ts=2026-04-28T00:02:21.050381988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-28T00:02:21.343648756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:46548" ts=2026-04-28T00:02:21.353579794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:46552" ts=2026-04-28T00:02:21.947207737Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-8854ca5d as failed, suspect timeout reached (2 peer confirmations)" level=debug ts=2026-04-28T00:02:36.12760665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:02:51.051264382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-28T00:03:21.052052588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cda7cbc5 10.128.0.85:7946" level=debug ts=2026-04-28T00:03:26.460448604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:03:29.8210671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:57280" ts=2026-04-28T00:03:36.00116566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:41766" ts=2026-04-28T00:03:51.053064936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-28T00:03:59.822574693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:46588" ts=2026-04-28T00:04:06.002990809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:38162" level=debug ts=2026-04-28T00:04:06.389458552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T00:04:06.390979083Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T00:04:21.053930619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=error ts=2026-04-28T00:04:42.349496242Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T00:04:42.351293755Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T00:04:42.353050921Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T00:04:42.353081749Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T00:04:51.055696907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-28T00:04:51.602573566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:46748" level=debug ts=2026-04-28T00:04:54.763025064Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:04:59.825602681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:43934" ts=2026-04-28T00:05:21.056687388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cda7cbc5 10.128.0.85:7946" ts=2026-04-28T00:05:51.057753526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cda7cbc5 10.128.0.85:7946" level=debug ts=2026-04-28T00:05:51.282438383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:05:51.605754807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:47790" ts=2026-04-28T00:06:06.008986437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:59134" ts=2026-04-28T00:06:21.058609194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-28T00:06:21.607049642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:46626" level=debug ts=2026-04-28T00:06:28.281102182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:06:29.829389437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:43422" ts=2026-04-28T00:06:36.009981408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:52220" ts=2026-04-28T00:06:51.060346286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-28T00:06:51.608662707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53882" level=debug ts=2026-04-28T00:07:06.124100062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:07:21.062770402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-28T00:07:51.064076575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cda7cbc5 10.128.0.85:7946" level=debug ts=2026-04-28T00:08:08.150421145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T00:08:08.151845832Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T00:08:21.065568196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-28T00:08:21.612595067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:44824" ts=2026-04-28T00:08:51.066719356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-28T00:08:51.613331149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:55574" level=debug ts=2026-04-28T00:08:55.108025663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:08:59.835222195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:57868" ts=2026-04-28T00:09:21.067717066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cda7cbc5 10.128.0.85:7946" ts=2026-04-28T00:09:21.614800436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:48276" level=debug ts=2026-04-28T00:09:47.852767286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:09:51.069137145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cda7cbc5 10.128.0.85:7946" ts=2026-04-28T00:09:59.837923515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:49408" ts=2026-04-28T00:10:06.018020721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:39904" ts=2026-04-28T00:10:21.070903016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-28T00:10:21.616877017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:48342" level=debug ts=2026-04-28T00:10:23.170710862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:10:36.019225891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:42858" ts=2026-04-28T00:10:51.072265548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cda7cbc5 10.128.0.85:7946" ts=2026-04-28T00:10:51.618555914Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:33762" ts=2026-04-28T00:10:59.840885356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:34424" ts=2026-04-28T00:11:21.073697756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-28T00:11:24.401010256Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:11:36.021038471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:46232" ts=2026-04-28T00:11:48.952139399Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-cda7cbc5 (timeout reached)" ts=2026-04-28T00:11:51.075102973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-28T00:11:51.952143565Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-cda7cbc5 has failed, no acks received" ts=2026-04-28T00:11:59.843661933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:42806" level=debug ts=2026-04-28T00:12:00.144465131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T00:12:00.146027834Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T00:12:11.952812608Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-cda7cbc5 as failed, suspect timeout reached (2 peer confirmations)" ts=2026-04-28T00:12:13.237291016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:57602" ts=2026-04-28T00:12:13.246693496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:57608" ts=2026-04-28T00:12:13.951634708Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-cda7cbc5 (timeout reached)" ts=2026-04-28T00:12:16.952390864Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-cda7cbc5 has failed, no acks received" ts=2026-04-28T00:12:21.076760419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-28T00:12:36.023144411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:50534" level=debug ts=2026-04-28T00:12:36.025116752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:12:51.077709983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b72584ce 10.128.0.85:7946" ts=2026-04-28T00:12:51.116419131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:40836" ts=2026-04-28T00:13:21.079067429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b72584ce 10.128.0.85:7946" ts=2026-04-28T00:13:21.117920867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:59394" level=debug ts=2026-04-28T00:13:22.751226453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:13:36.02645395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:44386" ts=2026-04-28T00:13:51.08017735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-28T00:13:51.119702653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:33100" level=debug ts=2026-04-28T00:14:20.672189483Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:14:21.081603919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b72584ce 10.128.0.85:7946" ts=2026-04-28T00:14:21.121186821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:44740" ts=2026-04-28T00:14:36.02948956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:58096" level=error ts=2026-04-28T00:14:42.34952198Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T00:14:42.351637592Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T00:14:42.353399275Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T00:14:42.353424895Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T00:14:51.08283285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b72584ce 10.128.0.85:7946" ts=2026-04-28T00:14:59.85274114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36332" ts=2026-04-28T00:15:06.030727902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:54280" ts=2026-04-28T00:15:21.08426072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-28T00:15:22.192820514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:15:29.85367634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:43180" ts=2026-04-28T00:15:36.032263425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:43802" ts=2026-04-28T00:15:51.086048545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-28T00:15:59.854983334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:43534" ts=2026-04-28T00:16:06.033879419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:50420" level=debug ts=2026-04-28T00:16:15.422844159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T00:16:15.424378462Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T00:16:21.087864091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-28T00:16:21.136425455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:32898" ts=2026-04-28T00:16:36.03492185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:49198" level=debug ts=2026-04-28T00:16:50.871016076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:16:51.093701554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-28T00:16:51.13867356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:52938" ts=2026-04-28T00:17:21.094464289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-28T00:17:36.036934232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:33888" level=debug ts=2026-04-28T00:17:49.899416876Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:17:51.095712622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-28T00:17:59.859868893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:47288" ts=2026-04-28T00:18:21.097098331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-28T00:18:23.952075685Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b72584ce (timeout reached)" ts=2026-04-28T00:18:26.951834972Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b72584ce has failed, no acks received" ts=2026-04-28T00:18:36.038792269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:46014" ts=2026-04-28T00:18:36.251074086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:35962" ts=2026-04-28T00:18:36.260536118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:35968" ts=2026-04-28T00:18:46.948105544Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-b72584ce as failed, suspect timeout reached (2 peer confirmations)" ts=2026-04-28T00:18:51.098134716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" level=debug ts=2026-04-28T00:18:51.59819901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:19:06.040488687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:44302" ts=2026-04-28T00:19:21.099281207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-28T00:19:34.812488946Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:19:36.042206283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:56438" ts=2026-04-28T00:19:51.100486828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-28T00:20:06.043631171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:51128" ts=2026-04-28T00:20:21.101521399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" level=debug ts=2026-04-28T00:20:23.103012949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T00:20:23.105423844Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T00:20:36.04509376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:59848" ts=2026-04-28T00:20:51.102975308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-28T00:20:59.609537524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:45902" ts=2026-04-28T00:20:59.867798478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:50136" level=debug ts=2026-04-28T00:21:09.030189949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:21:21.10434212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1747da48 10.128.0.85:7946" level=debug ts=2026-04-28T00:21:46.2404457Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:21:51.105291688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-28T00:21:59.612099985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:43642" ts=2026-04-28T00:22:21.106471904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" ts=2026-04-28T00:22:29.871085578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36048" level=debug ts=2026-04-28T00:22:43.890217102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:22:51.107346856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-28T00:22:59.614456688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:57754" ts=2026-04-28T00:23:21.108724574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mmn8n-884c9bc7 10.128.0.81:7946" level=debug ts=2026-04-28T00:23:40.813593872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T00:23:51.110386268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-m28j4-8cad8ef0 10.128.0.83:7946" ts=2026-04-28T00:23:59.87473308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:48316"