level=info ts=2026-04-28T02:41:03.061042265Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)" level=info ts=2026-04-28T02:41:03.061076578Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml level=info ts=2026-04-28T02:41:03.061906502Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095 level=debug ts=2026-04-28T02:41:03.062293238Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-04-28T02:41:03.062420458Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.89 inf=eth0 level=debug ts=2026-04-28T02:41:03.063804316Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-04-28T02:41:03.063909637Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.89 inf=eth0 level=info ts=2026-04-28T02:41:03.064537538Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-f307e64e ts=2026-04-28T02:41:03.064665669Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired" level=debug ts=2026-04-28T02:41:03.064682995Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.89 advertisePort=7946 level=debug ts=2026-04-28T02:41:03.064705279Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.89 advertisePort=7946 level=info ts=2026-04-28T02:41:03.064883982Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4 ts=2026-04-28T02:41:03.073249646Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=info ts=2026-04-28T02:41:03.073274995Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" level=warn ts=2026-04-28T02:41:03.073290143Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=8.408932ms level=error ts=2026-04-28T02:41:03.073300062Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure" level=info ts=2026-04-28T02:41:03.073317225Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local ts=2026-04-28T02:41:03.077339755Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-04-28T02:41:03.07737219Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" level=info ts=2026-04-28T02:41:03.081463424Z caller=module_service.go:82 msg=starting module=server level=debug ts=2026-04-28T02:41:03.081669541Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics level=debug ts=2026-04-28T02:41:03.081721641Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server level=info ts=2026-04-28T02:41:03.081729801Z caller=module_service.go:82 msg=starting module=memberlist-kv level=info ts=2026-04-28T02:41:03.08174349Z caller=module_service.go:82 msg=starting module=analytics level=info ts=2026-04-28T02:41:03.081866561Z caller=module_service.go:82 msg=starting module=runtime-config level=debug ts=2026-04-28T02:41:03.082182975Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv level=debug ts=2026-04-28T02:41:03.082196468Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config level=debug ts=2026-04-28T02:41:03.082213108Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server level=info ts=2026-04-28T02:41:03.082218479Z caller=module_service.go:82 msg=starting module=compactor level=info ts=2026-04-28T02:41:03.082261367Z caller=ring.go:365 msg="ring doesn't exist in KV store yet" level=info ts=2026-04-28T02:41:03.082301973Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor level=info ts=2026-04-28T02:41:03.082322892Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty" level=info ts=2026-04-28T02:41:03.082506917Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring" level=info ts=2026-04-28T02:41:03.082519152Z caller=compactor.go:387 msg="compactor is JOINING in the ring" level=debug ts=2026-04-28T02:41:03.082541402Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true level=info ts=2026-04-28T02:41:04.082645185Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring" ts=2026-04-28T02:41:04.187489395Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-04-28T02:41:04.187513789Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" level=info ts=2026-04-28T02:41:04.260766699Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring" level=info ts=2026-04-28T02:41:04.260813585Z caller=loki.go:599 msg="Loki started" startup_time=1.273409187s ts=2026-04-28T02:41:06.85975865Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-04-28T02:41:06.859786461Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" level=info ts=2026-04-28T02:41:09.261103958Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor" level=info ts=2026-04-28T02:41:09.261156063Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor" ts=2026-04-28T02:41:11.105202759Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-04-28T02:41:11.1052295Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" ts=2026-04-28T02:41:20.228905393Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-04-28T02:41:20.228930694Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" level=debug ts=2026-04-28T02:41:20.453162565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-28T02:41:30.151828376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:50848" ts=2026-04-28T02:41:34.210553781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:34492" level=debug ts=2026-04-28T02:41:36.660032328Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-28T02:41:45.379937286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:35422" ts=2026-04-28T02:41:49.937915927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:50532" ts=2026-04-28T02:41:51.537300478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T02:41:51.687363684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.129.0.88:7946" ts=2026-04-28T02:41:51.688640884Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.86:7946: dial tcp 10.129.0.86:7946: connect: connection refused" ts=2026-04-28T02:41:51.689515218Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.87:7946: dial tcp 10.129.0.87:7946: connect: connection refused" ts=2026-04-28T02:41:51.689625408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.129.0.89:7946" ts=2026-04-28T02:41:51.689694113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:48124" ts=2026-04-28T02:41:51.690215334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.129.0.85:7946" ts=2026-04-28T02:41:51.692176265Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.70:7946: dial tcp 10.130.0.70:7946: connect: connection refused" ts=2026-04-28T02:41:51.69333517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.130.0.71:7946" level=info ts=2026-04-28T02:41:51.694770895Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=48.621447714s level=debug ts=2026-04-28T02:42:05.159205556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-28T02:42:21.54029196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T02:42:25.662901661Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-28T02:42:30.97234323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:56646" ts=2026-04-28T02:42:51.541958234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T02:42:55.345890205Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=error ts=2026-04-28T02:42:55.347643127Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T02:43:00.975995619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:53426" ts=2026-04-28T02:43:21.543621452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T02:43:30.979152003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:50560" level=debug ts=2026-04-28T02:43:33.275277697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-28T02:43:34.22093237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:45408" ts=2026-04-28T02:43:51.545305248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T02:44:11.103671593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:35784" ts=2026-04-28T02:44:21.547715075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-971a9304 10.130.0.71:7946" ts=2026-04-28T02:44:30.985547048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:50366" level=debug ts=2026-04-28T02:44:39.143176523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-28T02:44:51.550343027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T02:45:04.228582757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:60598" ts=2026-04-28T02:45:21.552061273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T02:45:30.991469512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:43584" ts=2026-04-28T02:45:34.229457785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:43246" level=debug ts=2026-04-28T02:45:49.157854904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-28T02:45:51.552944365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T02:46:00.994085563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:55566" ts=2026-04-28T02:46:04.230499417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:50080" ts=2026-04-28T02:46:21.554218216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T02:46:41.114478856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:40690" level=debug ts=2026-04-28T02:46:47.828154409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-04-28T02:46:51.556772467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-971a9304 10.130.0.71:7946" ts=2026-04-28T02:47:21.558929655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T02:47:34.236925632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:48054" level=debug ts=2026-04-28T02:47:45.92050622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=error ts=2026-04-28T02:47:45.922128544Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T02:47:51.560624034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T02:48:21.563216554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-971a9304 10.130.0.71:7946" level=debug ts=2026-04-28T02:48:30.907787756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T02:48:51.565841661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T02:49:11.121834336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:57604" ts=2026-04-28T02:49:21.567260506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T02:49:23.700179155Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T02:49:34.246593141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:49614" ts=2026-04-28T02:49:51.570098244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-971a9304 10.130.0.71:7946" ts=2026-04-28T02:50:11.124307995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53750" level=debug ts=2026-04-28T02:50:14.177169527Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T02:50:21.572485317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T02:50:51.573983406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T02:51:04.253946684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:47406" level=info ts=2026-04-28T02:51:09.26133046Z caller=tables_manager.go:76 msg="compactor startup delay completed" level=error ts=2026-04-28T02:51:09.264799699Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T02:51:09.266555135Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T02:51:09.268176523Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T02:51:09.26821532Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=info ts=2026-04-28T02:51:09.26822936Z caller=tables_manager.go:145 msg="compactor started" ts=2026-04-28T02:51:11.128212685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:41634" level=debug ts=2026-04-28T02:51:15.095911731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T02:51:21.576226659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-971a9304 10.130.0.71:7946" ts=2026-04-28T02:51:34.25541508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:44850" level=debug ts=2026-04-28T02:51:50.641227379Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T02:51:50.642764882Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T02:51:51.578630495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T02:52:01.033043977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:40136" ts=2026-04-28T02:52:04.257033027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:55862" ts=2026-04-28T02:52:11.133165024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:58306" ts=2026-04-28T02:52:21.579987638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T02:52:34.258573368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:53020" level=debug ts=2026-04-28T02:52:39.127507798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T02:52:41.134300879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:33724" ts=2026-04-28T02:52:51.581833317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T02:53:04.259442514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:48420" ts=2026-04-28T02:53:11.135767386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53534" ts=2026-04-28T02:53:21.582691259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T02:53:33.513201056Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T02:53:34.260647653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:49306" ts=2026-04-28T02:53:51.584347096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T02:54:04.262410741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:53990" ts=2026-04-28T02:54:11.14053752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:43040" level=debug ts=2026-04-28T02:54:16.607463547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T02:54:21.586301722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-971a9304 10.130.0.71:7946" ts=2026-04-28T02:54:41.14171193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:56648" ts=2026-04-28T02:54:51.589060963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T02:55:03.730750303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T02:55:04.267494283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:55810" ts=2026-04-28T02:55:21.590501616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T02:55:49.877353706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T02:55:49.879653165Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T02:55:51.591532071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T02:56:04.273056062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:42948" ts=2026-04-28T02:56:11.150450689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:49004" ts=2026-04-28T02:56:21.592886923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T02:56:46.227954986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T02:56:51.594525888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T02:57:11.15617933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:56702" ts=2026-04-28T02:57:21.59575976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T02:57:30.809525063Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T02:57:34.278929886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:56240" ts=2026-04-28T02:57:41.157732914Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:35626" ts=2026-04-28T02:57:51.597558012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T02:58:09.826184653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T02:58:21.599826319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-971a9304 10.130.0.71:7946" level=debug ts=2026-04-28T02:58:48.392672454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T02:58:51.602942539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-971a9304 10.130.0.71:7946" ts=2026-04-28T02:59:01.077616723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:51214" ts=2026-04-28T02:59:04.288183546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:53396" ts=2026-04-28T02:59:11.163734996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:44184" ts=2026-04-28T02:59:21.606295257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-971a9304 10.130.0.71:7946" level=debug ts=2026-04-28T02:59:28.925305455Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T02:59:28.926927745Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T02:59:51.60858881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:00:04.290743269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:47048" level=debug ts=2026-04-28T03:00:19.815074409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:00:21.610343244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:00:51.612056648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:01:01.091082064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:34512" ts=2026-04-28T03:01:04.294257733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:39212" level=error ts=2026-04-28T03:01:09.272951666Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:01:09.275204525Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:01:09.277093447Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:01:09.277132659Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:01:11.17082654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:44312" level=debug ts=2026-04-28T03:01:21.585233979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:01:21.613513386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:01:34.295965688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:43716" ts=2026-04-28T03:01:51.615234676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:02:00.60002499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:02:04.297054874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:48210" ts=2026-04-28T03:02:21.616662446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:02:41.178714806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:49364" ts=2026-04-28T03:02:51.618376857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T03:02:54.711091881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:03:01.103753831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:38102" ts=2026-04-28T03:03:04.299628934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:48488" ts=2026-04-28T03:03:21.61995759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:03:34.301292914Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:36316" ts=2026-04-28T03:03:41.181832158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:55978" ts=2026-04-28T03:03:51.621139306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:03:53.217581325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T03:03:53.219209949Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:04:01.11001969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:35282" ts=2026-04-28T03:04:04.302600324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:52898" ts=2026-04-28T03:04:21.623009974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:04:41.184316423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:40378" ts=2026-04-28T03:04:51.624810256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:04:51.783870824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:05:01.116973887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47418" ts=2026-04-28T03:05:11.185975129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:45214" ts=2026-04-28T03:05:21.626425948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T03:05:37.263360629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:05:51.627747956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:06:04.309315547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:34018" level=debug ts=2026-04-28T03:06:15.257975839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:06:21.628586016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:06:51.629814758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:07:04.314114755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:38158" level=debug ts=2026-04-28T03:07:10.882578605Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:07:21.632469062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-971a9304 10.130.0.71:7946" ts=2026-04-28T03:07:31.133340835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:49526" ts=2026-04-28T03:07:51.635804679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-971a9304 10.130.0.71:7946" level=debug ts=2026-04-28T03:07:52.356810632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T03:07:52.35875678Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:08:11.204924887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53954" ts=2026-04-28T03:08:21.6389039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-971a9304 10.130.0.71:7946" ts=2026-04-28T03:08:31.139724787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:43846" ts=2026-04-28T03:08:34.317312757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:40472" ts=2026-04-28T03:08:51.640904273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T03:08:53.284804246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:09:21.6436215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-971a9304 10.130.0.71:7946" level=debug ts=2026-04-28T03:09:29.397133009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:09:31.147188272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:58566" ts=2026-04-28T03:09:51.646069015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:10:01.150303545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59700" ts=2026-04-28T03:10:02.810207013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59710" ts=2026-04-28T03:10:02.823896095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59712" ts=2026-04-28T03:10:15.065361993Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-971a9304 (timeout reached)" ts=2026-04-28T03:10:18.065209739Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-971a9304 has failed, no acks received" ts=2026-04-28T03:10:21.647287305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:10:22.287326767Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:10:32.873578651Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-971a9304 as failed, suspect timeout reached (2 peer confirmations)" ts=2026-04-28T03:10:34.326049008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:38668" ts=2026-04-28T03:10:51.649102006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:11:04.327485804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:47708" level=error ts=2026-04-28T03:11:09.278256387Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:11:09.281423296Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:11:09.283888902Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:11:09.283931557Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=debug ts=2026-04-28T03:11:16.690122873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:11:21.650897174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:11:51.652365949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T03:12:15.740296187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T03:12:15.741925016Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:12:21.653722681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:12:41.225574122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:46258" ts=2026-04-28T03:12:50.943351894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:49436" ts=2026-04-28T03:12:51.654709815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:13:04.334682508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:60868" level=debug ts=2026-04-28T03:13:16.17908107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:13:20.946335842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:34866" ts=2026-04-28T03:13:21.656057532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:13:51.658731639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fc8f1c8 10.130.0.71:7946" level=debug ts=2026-04-28T03:14:02.546243059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:14:11.236210991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:51710" ts=2026-04-28T03:14:21.660824427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:14:34.34333738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:54944" ts=2026-04-28T03:14:41.237616175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:60534" ts=2026-04-28T03:14:51.663045186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fc8f1c8 10.130.0.71:7946" level=debug ts=2026-04-28T03:15:03.36132252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:15:04.344891338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:44014" ts=2026-04-28T03:15:11.239002551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:49618" ts=2026-04-28T03:15:21.666454009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fc8f1c8 10.130.0.71:7946" ts=2026-04-28T03:15:50.963321835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:45522" ts=2026-04-28T03:15:51.673321178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fc8f1c8 10.130.0.71:7946" level=debug ts=2026-04-28T03:15:52.317903224Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:16:11.243442046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:58090" ts=2026-04-28T03:16:21.6808188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7fc8f1c8 10.130.0.71:7946" level=debug ts=2026-04-28T03:16:40.706470703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T03:16:40.708337258Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:16:51.685459237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:17:11.248539171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:33554" ts=2026-04-28T03:17:21.687493144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:17:37.904002681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:55728" ts=2026-04-28T03:17:37.917534617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:55744" level=debug ts=2026-04-28T03:17:39.035673149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:17:41.250065378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:38140" ts=2026-04-28T03:17:45.065558184Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-7fc8f1c8 (timeout reached)" ts=2026-04-28T03:17:48.065806576Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-7fc8f1c8 has failed, no acks received" ts=2026-04-28T03:17:51.688700626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:17:58.064703119Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-7fc8f1c8 as failed, suspect timeout reached (2 peer confirmations)" ts=2026-04-28T03:18:11.251614526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:59410" ts=2026-04-28T03:18:21.690002645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:18:30.475844005Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:18:34.359104794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:49780" ts=2026-04-28T03:18:41.253099344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:51372" ts=2026-04-28T03:18:51.691235124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T03:19:18.421440733Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:19:21.693773402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-be10396f 10.130.0.71:7946" ts=2026-04-28T03:19:34.364460292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:59064" ts=2026-04-28T03:19:51.697377752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-be10396f 10.130.0.71:7946" ts=2026-04-28T03:20:04.365873208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:55212" level=debug ts=2026-04-28T03:20:08.058186064Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:20:21.706192244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-be10396f 10.130.0.71:7946" level=debug ts=2026-04-28T03:20:50.271830247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T03:20:50.274014045Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:20:51.7088519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=error ts=2026-04-28T03:21:09.274258805Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:21:09.276268015Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:21:09.278465767Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:21:09.278502767Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:21:21.710911646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-be10396f 10.130.0.71:7946" ts=2026-04-28T03:21:34.372329599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:44748" level=debug ts=2026-04-28T03:21:51.451058936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:21:51.713991667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-be10396f 10.130.0.71:7946" ts=2026-04-28T03:21:56.440596794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:58638" ts=2026-04-28T03:22:21.716483095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:22:38.877238895Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:22:41.273908056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:44450" ts=2026-04-28T03:22:51.718275996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:23:04.380262435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:51414" ts=2026-04-28T03:23:21.720905271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-be10396f 10.130.0.71:7946" level=debug ts=2026-04-28T03:23:35.058968914Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:23:51.723377323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:23:56.461715876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:34840" ts=2026-04-28T03:24:21.725208881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:24:26.46519065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:49822" level=debug ts=2026-04-28T03:24:36.080662752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:24:51.727276976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-be10396f 10.130.0.71:7946" ts=2026-04-28T03:25:04.388701697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:53792" level=debug ts=2026-04-28T03:25:18.005116507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T03:25:18.006560635Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:25:21.730555088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-be10396f 10.130.0.71:7946" ts=2026-04-28T03:25:41.286075025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53360" ts=2026-04-28T03:25:51.734355724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-be10396f 10.130.0.71:7946" ts=2026-04-28T03:26:04.393801982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:33550" ts=2026-04-28T03:26:11.287143741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:43234" ts=2026-04-28T03:26:15.066256425Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-be10396f (timeout reached)" level=debug ts=2026-04-28T03:26:17.129896624Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:26:18.065793591Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-be10396f has failed, no acks received" ts=2026-04-28T03:26:19.888545386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59114" ts=2026-04-28T03:26:19.90220165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59122" ts=2026-04-28T03:26:21.737608245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c290147 10.130.0.71:7946" ts=2026-04-28T03:26:28.063634381Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-be10396f as failed, suspect timeout reached (2 peer confirmations)" ts=2026-04-28T03:26:34.395033246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:41258" ts=2026-04-28T03:26:51.739942863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T03:26:54.350195993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:27:11.291547366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:57792" ts=2026-04-28T03:27:21.741844816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:27:41.292730434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:45662" level=debug ts=2026-04-28T03:27:48.643357391Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:27:51.743261753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:28:04.400739874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:33840" ts=2026-04-28T03:28:11.293897389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:37762" ts=2026-04-28T03:28:21.746112146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c290147 10.130.0.71:7946" ts=2026-04-28T03:28:27.153899504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:52724" ts=2026-04-28T03:28:34.401985268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:56340" level=debug ts=2026-04-28T03:28:41.676282668Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:28:51.749327802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c290147 10.130.0.71:7946" ts=2026-04-28T03:29:11.296606739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:39614" ts=2026-04-28T03:29:21.751652439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:29:27.160573395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:50434" level=debug ts=2026-04-28T03:29:43.233083341Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T03:29:43.234615316Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:29:51.754042494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c290147 10.130.0.71:7946" ts=2026-04-28T03:29:57.163297722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:52664" ts=2026-04-28T03:30:21.756647454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:30:34.411722261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:33148" level=debug ts=2026-04-28T03:30:43.402173151Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:30:51.757678323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:30:57.169433319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:60050" level=error ts=2026-04-28T03:31:09.272304717Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:31:09.274209403Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:31:09.276000553Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:31:09.276032644Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:31:21.759332543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:31:43.017079461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:31:51.760608601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:31:57.175322096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:48000" ts=2026-04-28T03:32:11.312364321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:40096" ts=2026-04-28T03:32:21.761815802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T03:32:25.048501635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:32:34.420564704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:48736" ts=2026-04-28T03:32:41.313945008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:42392" ts=2026-04-28T03:32:51.763377489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:32:57.181204998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:41854" level=debug ts=2026-04-28T03:33:07.192949223Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:33:11.315411195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:58852" ts=2026-04-28T03:33:21.765031509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:33:34.422983822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:32960" ts=2026-04-28T03:33:40.065696815Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-2c290147 (timeout reached)" ts=2026-04-28T03:33:43.06595453Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-2c290147 has failed, no acks received" ts=2026-04-28T03:33:45.066916676Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-2c290147 (timeout reached)" ts=2026-04-28T03:33:48.06656227Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-2c290147 has failed, no acks received" ts=2026-04-28T03:33:51.766218613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:34:04.424027569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:35232" level=debug ts=2026-04-28T03:34:07.610241904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T03:34:07.611971401Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:34:10.887189116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:41016" ts=2026-04-28T03:34:10.901525862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:41018" ts=2026-04-28T03:34:21.76859412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6abafeb 10.130.0.71:7946" ts=2026-04-28T03:34:41.321734619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:37330" ts=2026-04-28T03:34:51.771178947Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T03:35:00.248224475Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:35:11.323359901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:60092" ts=2026-04-28T03:35:21.773518343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6abafeb 10.130.0.71:7946" level=debug ts=2026-04-28T03:35:39.172901876Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:35:51.77608958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:36:11.326701978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:39370" ts=2026-04-28T03:36:21.778094809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T03:36:23.274368989Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:36:45.537618613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:36436" ts=2026-04-28T03:36:51.779259306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T03:37:02.866344947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:37:11.331322406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:44644" ts=2026-04-28T03:37:21.780654077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:37:34.439002869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:44558" ts=2026-04-28T03:37:41.332516656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:32920" ts=2026-04-28T03:37:51.782429854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:38:03.475135794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T03:38:03.476861272Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:38:21.783837943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:38:39.499417691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:38:51.784851302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:39:21.786074242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:39:21.931874723Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:39:34.449221828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:60102" ts=2026-04-28T03:39:45.557132706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:34194" ts=2026-04-28T03:39:51.787443966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T03:40:18.024357657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:40:21.789823475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6abafeb 10.130.0.71:7946" ts=2026-04-28T03:40:51.792180901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:41:03.702152638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T03:41:09.271039209Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:41:09.272580972Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:41:09.274036579Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:41:09.274064225Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:41:21.793375954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:41:45.571002334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:39816" ts=2026-04-28T03:41:51.794805223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T03:42:01.040047222Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T03:42:01.041657529Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:42:04.458103917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:54238" ts=2026-04-28T03:42:05.065253985Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-f6abafeb (timeout reached)" ts=2026-04-28T03:42:08.065944995Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-f6abafeb has failed, no acks received" ts=2026-04-28T03:42:11.349733152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:41846" ts=2026-04-28T03:42:20.066785527Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-f6abafeb (timeout reached)" ts=2026-04-28T03:42:21.795989965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:42:23.066148972Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-f6abafeb has failed, no acks received" ts=2026-04-28T03:42:41.351175141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:50352" ts=2026-04-28T03:42:51.796887185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:42:53.776559164Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:43:04.460098411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:56064" ts=2026-04-28T03:43:11.352686617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53514" ts=2026-04-28T03:43:21.798036651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:43:24.898941805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:49050" ts=2026-04-28T03:43:24.91319273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:49054" ts=2026-04-28T03:43:34.461958092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:47572" level=debug ts=2026-04-28T03:43:51.49568769Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:43:51.799847179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:44:04.463401682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:58396" ts=2026-04-28T03:44:21.801453028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:44:38.086205915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:44:51.803689433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22156c03 10.130.0.71:7946" level=debug ts=2026-04-28T03:45:12.135369502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:45:21.805817426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:45:34.468032369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:42874" ts=2026-04-28T03:45:35.064462237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:53446" ts=2026-04-28T03:45:41.361347493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:43244" ts=2026-04-28T03:45:51.807209086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:45:55.513629886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T03:45:55.516451942Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:46:05.068015862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:50150" ts=2026-04-28T03:46:21.808355687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T03:46:33.877653805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:46:51.809299122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:47:04.473279324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:34616" ts=2026-04-28T03:47:05.07459873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:44826" ts=2026-04-28T03:47:11.367057633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:46946" ts=2026-04-28T03:47:21.810659095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:47:32.428904112Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:47:41.368355414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:55086" ts=2026-04-28T03:47:51.813511921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22156c03 10.130.0.71:7946" level=debug ts=2026-04-28T03:48:09.355697851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:48:11.370204484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:37864" ts=2026-04-28T03:48:21.815458263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:48:34.478444517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:57896" ts=2026-04-28T03:48:51.81729569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:48:54.79405493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:49:04.479398828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:51454" ts=2026-04-28T03:49:11.373592379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:43948" ts=2026-04-28T03:49:21.819033844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:49:34.481176677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:39770" level=debug ts=2026-04-28T03:49:40.909714639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T03:49:40.911714036Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:49:51.82165174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22156c03 10.130.0.71:7946" ts=2026-04-28T03:50:04.482496458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:48302" ts=2026-04-28T03:50:21.823778881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T03:50:37.737746884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:50:51.825399431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:51:05.100032302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:37212" level=error ts=2026-04-28T03:51:09.274279636Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:51:09.276377827Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:51:09.278147607Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T03:51:09.278177449Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=debug ts=2026-04-28T03:51:18.836431613Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:51:21.827332759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-22156c03 10.130.0.71:7946" ts=2026-04-28T03:51:35.103567936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:49990" ts=2026-04-28T03:51:50.0661783Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-22156c03 (timeout reached)" ts=2026-04-28T03:51:51.829715728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:51:53.066130224Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-22156c03 has failed, no acks received" ts=2026-04-28T03:51:55.066239215Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-22156c03 (timeout reached)" ts=2026-04-28T03:51:58.066519838Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-22156c03 has failed, no acks received" level=debug ts=2026-04-28T03:52:05.71781832Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:52:21.830666866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:52:34.491347413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:46310" ts=2026-04-28T03:52:51.832588819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:53:03.513672742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:53:04.492944018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:45732" ts=2026-04-28T03:53:11.388925607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:48714" ts=2026-04-28T03:53:21.833606778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:53:26.882607165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59816" ts=2026-04-28T03:53:26.895607073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59818" level=debug ts=2026-04-28T03:53:39.91117477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T03:53:39.913077075Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:53:51.835425882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:54:11.393871826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:51240" ts=2026-04-28T03:54:21.837978256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4894d19f 10.130.0.71:7946" level=debug ts=2026-04-28T03:54:26.927981294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:54:39.747739374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:38538" ts=2026-04-28T03:54:51.840964868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4894d19f 10.130.0.71:7946" ts=2026-04-28T03:55:21.843292932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T03:55:23.397036091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:55:34.502889722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:42966" ts=2026-04-28T03:55:39.753897344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47504" ts=2026-04-28T03:55:51.844519414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:56:04.504294333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:45326" ts=2026-04-28T03:56:09.757143548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:35610" ts=2026-04-28T03:56:11.402183168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:39744" level=debug ts=2026-04-28T03:56:17.726178606Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:56:21.845877282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:56:34.505313017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:46040" ts=2026-04-28T03:56:39.759881217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:42010" ts=2026-04-28T03:56:51.847097897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:57:04.506639435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:55382" ts=2026-04-28T03:57:09.763251104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:39452" ts=2026-04-28T03:57:11.406820589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:43670" level=debug ts=2026-04-28T03:57:16.573763647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:57:21.84882696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T03:57:51.850099148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:57:53.381652122Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T03:57:53.38322232Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T03:58:21.851222284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T03:58:32.886098757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:58:34.514746629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:34736" ts=2026-04-28T03:58:51.852516558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:59:09.774941833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:34426" level=debug ts=2026-04-28T03:59:18.575932179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T03:59:21.853949052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T03:59:51.855772038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4894d19f 10.130.0.71:7946" ts=2026-04-28T04:00:04.522876727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:45340" level=debug ts=2026-04-28T04:00:09.825095076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:00:11.41895908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:60444" ts=2026-04-28T04:00:21.858456736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T04:00:35.066658582Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-4894d19f (timeout reached)" ts=2026-04-28T04:00:38.065743298Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-4894d19f has failed, no acks received" ts=2026-04-28T04:00:45.06636751Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-4894d19f (timeout reached)" ts=2026-04-28T04:00:48.066313812Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-4894d19f has failed, no acks received" ts=2026-04-28T04:00:51.111767924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:46372" ts=2026-04-28T04:00:51.124308387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:46374" ts=2026-04-28T04:00:51.859899296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T04:00:55.06583596Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-4894d19f (timeout reached)" ts=2026-04-28T04:00:58.065439454Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-4894d19f as failed, suspect timeout reached (2 peer confirmations)" ts=2026-04-28T04:00:58.065471005Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-4894d19f has failed, no acks received" level=error ts=2026-04-28T04:01:09.27134803Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T04:01:09.273008721Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T04:01:09.27476376Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T04:01:09.274796723Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T04:01:11.422271991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:56594" level=debug ts=2026-04-28T04:01:11.468710651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:01:21.861293109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T04:01:34.527634251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:40558" ts=2026-04-28T04:01:41.423584597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:50156" ts=2026-04-28T04:01:51.863105868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T04:01:57.887996972Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T04:01:57.890080858Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T04:02:21.865376022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-39d8a8fb 10.130.0.71:7946" level=debug ts=2026-04-28T04:02:43.643927898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:02:51.867912225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T04:03:04.533101886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:36626" ts=2026-04-28T04:03:07.787522438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:44310" ts=2026-04-28T04:03:21.870020999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T04:03:29.984089017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:03:34.534274855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:56942" ts=2026-04-28T04:03:51.871133963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T04:04:20.116727491Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:04:21.872581095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T04:04:51.87478538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-39d8a8fb 10.130.0.71:7946" level=debug ts=2026-04-28T04:04:59.975845914Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:05:04.54018154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:55474" ts=2026-04-28T04:05:21.87668968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T04:05:37.803198475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59514" level=debug ts=2026-04-28T04:05:38.177951921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T04:05:38.179676055Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T04:05:51.879343783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-39d8a8fb 10.130.0.71:7946" ts=2026-04-28T04:06:07.805927482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:60674" ts=2026-04-28T04:06:21.881630224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T04:06:31.640844226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:06:37.808642625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:38178" ts=2026-04-28T04:06:51.882669389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T04:07:04.546592194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:54796" level=debug ts=2026-04-28T04:07:10.764635031Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:07:21.885252942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-39d8a8fb 10.130.0.71:7946" ts=2026-04-28T04:07:51.887845492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T04:07:55.026667928Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:08:21.888795015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T04:08:34.553351703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:60236" level=debug ts=2026-04-28T04:08:47.720605345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:08:50.066630829Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-39d8a8fb (timeout reached)" ts=2026-04-28T04:08:51.890363621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T04:08:53.065674741Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-39d8a8fb has failed, no acks received" ts=2026-04-28T04:09:00.882025938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47088" ts=2026-04-28T04:09:00.895544141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47102" ts=2026-04-28T04:09:05.06642887Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-39d8a8fb (timeout reached)" ts=2026-04-28T04:09:08.066227796Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-39d8a8fb has failed, no acks received" ts=2026-04-28T04:09:08.074064817Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-39d8a8fb as failed, suspect timeout reached (2 peer confirmations)" ts=2026-04-28T04:09:21.892320997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T04:09:34.555612012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:44964" level=debug ts=2026-04-28T04:09:35.677862922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T04:09:35.679351209Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T04:09:41.461758446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:49692" ts=2026-04-28T04:09:51.893499896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T04:10:11.462968006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:41326" ts=2026-04-28T04:10:21.895218239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T04:10:26.61201277Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:10:51.896336959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T04:11:04.148609436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:51970" level=debug ts=2026-04-28T04:11:08.089099951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T04:11:09.271280537Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T04:11:09.272971582Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T04:11:09.274562011Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T04:11:09.274592284Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T04:11:11.465691285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:37896" ts=2026-04-28T04:11:21.898129039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T04:11:34.151666221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:33156" level=debug ts=2026-04-28T04:11:50.084433854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:11:51.899652507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T04:12:11.470092396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:46982" ts=2026-04-28T04:12:21.900698351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T04:12:34.158420247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:48286" ts=2026-04-28T04:12:41.471672656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:37806" level=debug ts=2026-04-28T04:12:46.694322595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:12:51.903318949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f028d2d3 10.130.0.71:7946" ts=2026-04-28T04:13:04.162038826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:49142" ts=2026-04-28T04:13:04.573784497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:55558" level=debug ts=2026-04-28T04:13:21.694089098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T04:13:21.695874728Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T04:13:21.90571614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T04:13:51.90730857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" level=debug ts=2026-04-28T04:14:07.366650622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:14:11.477000601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:44562" ts=2026-04-28T04:14:21.910215066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f028d2d3 10.130.0.71:7946" level=debug ts=2026-04-28T04:14:47.657256961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:14:51.912003081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T04:15:04.175240272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:55484" ts=2026-04-28T04:15:04.581045642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:55940" ts=2026-04-28T04:15:21.914120905Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-f028d2d3 failed: dial tcp 10.130.0.71:7946: connect: connection refused" level=debug ts=2026-04-28T04:15:24.261940893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:15:25.065127157Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-f028d2d3 (timeout reached)" ts=2026-04-28T04:15:28.065265952Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-f028d2d3 has failed, no acks received" ts=2026-04-28T04:15:35.065821318Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-f028d2d3 (timeout reached)" ts=2026-04-28T04:15:38.065486165Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-f028d2d3 has failed, no acks received" ts=2026-04-28T04:15:41.482896694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:56568" ts=2026-04-28T04:15:51.91503227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T04:15:56.896725299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:57494" ts=2026-04-28T04:15:56.910200181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:57508" level=debug ts=2026-04-28T04:16:02.552549571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:16:21.917279592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3e915bf4 10.130.0.71:7946" ts=2026-04-28T04:16:29.175341059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:41768" level=debug ts=2026-04-28T04:16:47.354965381Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T04:16:47.356767417Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T04:16:51.920938828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3e915bf4 10.130.0.71:7946" ts=2026-04-28T04:17:04.589033328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:33042" ts=2026-04-28T04:17:11.486438461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:50822" ts=2026-04-28T04:17:21.923238737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T04:17:37.342889831Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:17:41.488358246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:59296" ts=2026-04-28T04:17:51.926003356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3e915bf4 10.130.0.71:7946" ts=2026-04-28T04:17:59.185761843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:58530" ts=2026-04-28T04:18:11.490144766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:43136" level=debug ts=2026-04-28T04:18:20.335943732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:18:21.928393872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" ts=2026-04-28T04:18:41.491883466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:36286" ts=2026-04-28T04:18:51.929418517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T04:19:11.492872343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53522" level=debug ts=2026-04-28T04:19:13.078887858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:19:21.931034783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T04:19:29.195812899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:60804" ts=2026-04-28T04:19:51.93234155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=debug ts=2026-04-28T04:19:55.47111001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-04-28T04:19:59.199112992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59200" ts=2026-04-28T04:20:04.603888627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:44440" ts=2026-04-28T04:20:21.933626699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mk7xh-17d015f6 10.129.0.88:7946" ts=2026-04-28T04:20:34.605569024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.88:48794" level=debug ts=2026-04-28T04:20:37.218417064Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-04-28T04:20:37.219980979Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-04-28T04:20:41.496695668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:39198" ts=2026-04-28T04:20:51.935033132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fwxrk-4e256c9b 10.129.0.85:7946" level=error ts=2026-04-28T04:21:09.272226955Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T04:21:09.274106835Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T04:21:09.276033311Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-04-28T04:21:09.27605846Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"