level=info ts=2026-04-27T11:08:33.811217278Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-27T11:08:33.811277103Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-27T11:08:33.812775796Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-27T11:08:33.813103507Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-27T11:08:33.813176912Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.72 inf=eth0
level=debug ts=2026-04-27T11:08:33.814000602Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-27T11:08:33.814081227Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.72 inf=eth0
level=info ts=2026-04-27T11:08:33.814116513Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-e58b37cd
ts=2026-04-27T11:08:33.814185989Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-27T11:08:33.814199597Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.72 advertisePort=7946
level=debug ts=2026-04-27T11:08:33.814216556Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.72 advertisePort=7946
level=info ts=2026-04-27T11:08:33.814352458Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=info ts=2026-04-27T11:08:33.82746366Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-27T11:08:33.827491432Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-27T11:08:33.827513491Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-27T11:08:33.827512985Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-27T11:08:33.827537457Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-27T11:08:33.82754225Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-27T11:08:33.827600583Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-27T11:08:33.827622153Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-27T11:08:33.827876169Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-27T11:08:33.827888131Z caller=module_service.go:82 msg=starting module=compactor
ts=2026-04-27T11:08:33.836908966Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-27T11:08:33.836925202Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-27T11:08:33.836937062Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=22.586438ms
level=error ts=2026-04-27T11:08:33.836946347Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-27T11:08:33.836963528Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-04-27T11:08:33.836995273Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-27T11:08:33.837033982Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-27T11:08:33.837235056Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-27T11:08:33.837246845Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-27T11:08:33.83726221Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-04-27T11:08:33.839834264Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-27T11:08:33.839846829Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-27T11:08:34.837397104Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-27T11:08:34.970615727Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-27T11:08:34.970670286Z caller=loki.go:599 msg="Loki started" startup_time=1.219413622s
ts=2026-04-27T11:08:35.490929863Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-27T11:08:35.490968344Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T11:08:38.491003227Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-27T11:08:38.49102302Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-27T11:08:39.971005264Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-27T11:08:39.971054296Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-27T11:08:46.006445299Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-27T11:08:46.006467638Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-27T11:08:47.97767785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T11:08:57.862757912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:48866"
ts=2026-04-27T11:08:58.508878892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:39646"
ts=2026-04-27T11:08:59.942261445Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.74:7946: dial tcp 10.130.0.74:7946: connect: connection refused"
ts=2026-04-27T11:08:59.942373066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.72:7946"
ts=2026-04-27T11:08:59.942391847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:40624"
level=info ts=2026-04-27T11:08:59.942799816Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=26.105830554s
ts=2026-04-27T11:09:01.691257998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:56300"
level=debug ts=2026-04-27T11:09:09.371790206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T11:09:18.948111055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:54370"
ts=2026-04-27T11:09:22.742355404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T11:09:28.631762736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T11:09:39.570267019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:35560"
ts=2026-04-27T11:09:48.951596401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:43230"
ts=2026-04-27T11:09:52.745601312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:09:54.932068811Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T11:10:15.650255002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:50808"
ts=2026-04-27T11:10:18.95529378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:58668"
ts=2026-04-27T11:10:22.749222265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:10:23.289871564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-27T11:10:23.291425806Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T11:10:39.575666552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:48646"
ts=2026-04-27T11:10:52.752341971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T11:11:09.579174359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:55968"
level=debug ts=2026-04-27T11:11:10.322776451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T11:11:18.962173752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:57536"
ts=2026-04-27T11:11:22.755628827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T11:11:39.582921197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:53836"
ts=2026-04-27T11:11:52.75867381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T11:12:09.586701928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:53316"
level=debug ts=2026-04-27T11:12:15.047394647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T11:12:18.966044007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:46134"
ts=2026-04-27T11:12:22.76230024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:12:52.764446973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90fce612 10.130.0.73:7946"
level=debug ts=2026-04-27T11:13:21.656597085Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T11:13:22.765586809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90fce612 10.130.0.73:7946"
ts=2026-04-27T11:13:52.767959365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:14:09.597614172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:33698"
ts=2026-04-27T11:14:15.677275538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:52774"
ts=2026-04-27T11:14:22.771223748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T11:14:25.66001845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T11:14:48.9835793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:53858"
ts=2026-04-27T11:14:52.774188391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:15:21.888712287Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-27T11:15:21.890541023Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T11:15:22.77718472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:15:45.685215192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:48750"
ts=2026-04-27T11:15:52.780942261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:16:01.805240915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:16:22.783384029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90fce612 10.130.0.73:7946"
level=debug ts=2026-04-27T11:16:38.667296975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:16:39.613341819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:37368"
ts=2026-04-27T11:16:45.690520773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:51960"
ts=2026-04-27T11:16:52.784835437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90fce612 10.130.0.73:7946"
ts=2026-04-27T11:17:22.787398067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:17:35.249530403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:17:45.695332126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:45992"
ts=2026-04-27T11:17:48.99971335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:46796"
ts=2026-04-27T11:17:52.790282722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T11:18:22.792655598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90fce612 10.130.0.73:7946"
level=debug ts=2026-04-27T11:18:36.028294417Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-27T11:18:39.971927686Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-27T11:18:39.974527606Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T11:18:39.976666935Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T11:18:39.978646504Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T11:18:39.978677799Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-27T11:18:39.978691092Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-27T11:18:45.70061568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:47688"
ts=2026-04-27T11:18:49.00644738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:37672"
ts=2026-04-27T11:18:52.795491499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:19:09.627619419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:33988"
ts=2026-04-27T11:19:22.798749135Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T11:19:23.498586599Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T11:19:23.500552646Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T11:19:52.80111958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90fce612 10.130.0.73:7946"
level=debug ts=2026-04-27T11:20:13.351876041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:20:22.803577999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T11:20:39.635771552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:38944"
ts=2026-04-27T11:20:45.71290601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:45386"
ts=2026-04-27T11:20:52.816806014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:21:05.62751281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:21:09.638905467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:59602"
ts=2026-04-27T11:21:22.822416843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:21:47.639350858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:21:52.824360405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90fce612 10.130.0.73:7946"
ts=2026-04-27T11:22:09.643646925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:60402"
ts=2026-04-27T11:22:19.029007347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:49600"
ts=2026-04-27T11:22:22.826991338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:22:41.507292153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:22:52.83060209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:23:09.651237723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:46756"
ts=2026-04-27T11:23:15.728248308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:59938"
ts=2026-04-27T11:23:22.833882007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:23:36.955555469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T11:23:36.957636907Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T11:23:52.836041503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90fce612 10.130.0.73:7946"
ts=2026-04-27T11:24:09.661908167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:52558"
ts=2026-04-27T11:24:19.040727799Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:52134"
ts=2026-04-27T11:24:22.838775029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T11:24:37.123468988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:24:39.665881197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:40800"
ts=2026-04-27T11:24:45.737253229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:34602"
ts=2026-04-27T11:24:49.044505218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:49978"
ts=2026-04-27T11:24:52.841000835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90fce612 10.130.0.73:7946"
ts=2026-04-27T11:25:15.73885651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:38960"
ts=2026-04-27T11:25:19.047832147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:41194"
level=debug ts=2026-04-27T11:25:20.615491502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:25:22.843638694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:25:52.846821009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:26:17.053238343Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:26:19.054697915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:50788"
ts=2026-04-27T11:26:22.850214122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T11:26:52.532882358Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:26:52.853444371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:27:22.855323341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90fce612 10.130.0.73:7946"
level=debug ts=2026-04-27T11:27:28.722566012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T11:27:28.724195879Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T11:27:49.062577492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:59000"
ts=2026-04-27T11:27:52.858009685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:28:19.066050843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:36776"
ts=2026-04-27T11:28:22.861753484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:28:29.380586435Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T11:28:39.98309515Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T11:28:39.985282134Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T11:28:39.987192017Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T11:28:39.987217262Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T11:28:52.865537883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T11:29:09.062478725Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:29:22.869142244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T11:29:45.769028245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:35116"
level=debug ts=2026-04-27T11:29:48.344467803Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:29:52.872358069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T11:30:15.770431684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:56410"
ts=2026-04-27T11:30:22.875730371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:30:28.016640369Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:30:49.079830384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:33366"
ts=2026-04-27T11:30:52.878440218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90fce612 10.130.0.73:7946"
ts=2026-04-27T11:31:15.775493055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:56258"
level=debug ts=2026-04-27T11:31:21.286627243Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T11:31:21.288626915Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T11:31:22.88081674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T11:31:45.776761253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:46204"
ts=2026-04-27T11:31:49.08706607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:56836"
ts=2026-04-27T11:31:52.88374235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:32:06.984253858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:32:19.090936492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:49110"
ts=2026-04-27T11:32:22.8876472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:32:52.891653527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:33:02.856663046Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:33:19.549962513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:37842"
ts=2026-04-27T11:33:19.563389226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:37852"
ts=2026-04-27T11:33:22.895675468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:33:45.815817482Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-90fce612 (timeout reached)"
ts=2026-04-27T11:33:48.81563397Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-90fce612 has failed, no acks received"
level=debug ts=2026-04-27T11:33:52.468966397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:33:52.89885303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:34:19.099865702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:44524"
ts=2026-04-27T11:34:22.901014512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-82ec2083 10.130.0.73:7946"
level=debug ts=2026-04-27T11:34:27.140565227Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:34:52.903094856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T11:35:17.935589362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T11:35:17.93748907Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T11:35:22.905181525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-82ec2083 10.130.0.73:7946"
ts=2026-04-27T11:35:39.710477838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:59040"
ts=2026-04-27T11:35:52.906274577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-82ec2083 10.130.0.73:7946"
level=debug ts=2026-04-27T11:35:57.300071305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:36:19.108753696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:60456"
ts=2026-04-27T11:36:22.908251556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T11:36:30.902160838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:54666"
level=debug ts=2026-04-27T11:36:33.656423592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:36:39.718223463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:49114"
ts=2026-04-27T11:36:52.911788389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:37:09.721445251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:34952"
ts=2026-04-27T11:37:22.913764835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-82ec2083 10.130.0.73:7946"
level=debug ts=2026-04-27T11:37:31.735162957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:37:39.724496593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:54752"
ts=2026-04-27T11:37:52.916022217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:38:00.910177562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:58236"
level=debug ts=2026-04-27T11:38:18.550761846Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:38:22.919874014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T11:38:30.911354138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:46912"
ts=2026-04-27T11:38:39.732150458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:35978"
level=error ts=2026-04-27T11:38:39.984298642Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T11:38:39.987338679Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T11:38:39.99013971Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T11:38:39.990182706Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T11:38:52.924111309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:38:54.298792667Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T11:38:54.305714715Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T11:39:09.736242971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:52014"
ts=2026-04-27T11:39:19.131859836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:48940"
ts=2026-04-27T11:39:22.92716959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-82ec2083 10.130.0.73:7946"
ts=2026-04-27T11:39:30.916124621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:50464"
level=debug ts=2026-04-27T11:39:39.950874745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:39:52.929672061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:40:22.932553038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-82ec2083 10.130.0.73:7946"
level=debug ts=2026-04-27T11:40:39.829766251Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:40:52.934017434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-82ec2083 10.130.0.73:7946"
ts=2026-04-27T11:41:00.924664856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:38942"
level=debug ts=2026-04-27T11:41:20.895367819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:41:22.936034658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:41:30.926722745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:37288"
ts=2026-04-27T11:41:50.814869847Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-82ec2083 (timeout reached)"
ts=2026-04-27T11:41:52.939398358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:41:53.814929064Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-82ec2083 has failed, no acks received"
ts=2026-04-27T11:42:00.816019568Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-82ec2083 (timeout reached)"
ts=2026-04-27T11:42:02.382117402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:52492"
ts=2026-04-27T11:42:02.399283225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:52496"
level=debug ts=2026-04-27T11:42:02.874029229Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:42:03.816092141Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-82ec2083 has failed, no acks received"
ts=2026-04-27T11:42:22.942635764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T11:42:38.103426882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T11:42:38.1056334Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T11:42:52.945444685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23755843 10.130.0.73:7946"
ts=2026-04-27T11:43:09.760608322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:59366"
ts=2026-04-27T11:43:22.94769397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:43:28.446734793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:43:49.15744875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:53752"
ts=2026-04-27T11:43:52.949779716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23755843 10.130.0.73:7946"
level=debug ts=2026-04-27T11:44:17.420242663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:44:22.951213064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23755843 10.130.0.73:7946"
ts=2026-04-27T11:44:39.766160442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:41766"
ts=2026-04-27T11:44:49.164449818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:40496"
ts=2026-04-27T11:44:52.952936379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23755843 10.130.0.73:7946"
level=debug ts=2026-04-27T11:45:05.80646137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:45:22.954027862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23755843 10.130.0.73:7946"
ts=2026-04-27T11:45:38.978553219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:57756"
level=debug ts=2026-04-27T11:45:44.011517585Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:45:52.956591661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:46:09.774659077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:39382"
ts=2026-04-27T11:46:22.959626295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:46:27.550987547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T11:46:27.552776119Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T11:46:49.17905572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:34688"
ts=2026-04-27T11:46:52.963199674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:47:19.182142541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:36798"
ts=2026-04-27T11:47:22.966684799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:47:29.069122159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:47:52.969087435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23755843 10.130.0.73:7946"
level=debug ts=2026-04-27T11:48:21.015300985Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:48:22.97131285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=error ts=2026-04-27T11:48:39.982616049Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T11:48:39.984809747Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T11:48:39.986944609Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T11:48:39.986971237Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T11:48:52.973685972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23755843 10.130.0.73:7946"
level=debug ts=2026-04-27T11:48:58.149009991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:49:22.975740717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T11:49:39.00416504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:55238"
ts=2026-04-27T11:49:39.793545427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:44072"
ts=2026-04-27T11:49:52.97856825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T11:49:55.248323372Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:50:09.005343543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:38472"
ts=2026-04-27T11:50:09.796905854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:41216"
ts=2026-04-27T11:50:19.195594111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:43712"
ts=2026-04-27T11:50:22.982296344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:50:39.007400753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:49594"
ts=2026-04-27T11:50:39.800575303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:38858"
ts=2026-04-27T11:50:49.198778861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:43326"
level=debug ts=2026-04-27T11:50:52.096428078Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T11:50:52.097998315Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T11:50:52.985817143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T11:51:22.987820067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23755843 10.130.0.73:7946"
ts=2026-04-27T11:51:30.814872467Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-23755843 (timeout reached)"
ts=2026-04-27T11:51:33.814700681Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-23755843 has failed, no acks received"
level=debug ts=2026-04-27T11:51:41.199860545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:51:45.815601115Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-23755843 (timeout reached)"
ts=2026-04-27T11:51:48.816311521Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-23755843 has failed, no acks received"
ts=2026-04-27T11:51:52.9698507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:43240"
ts=2026-04-27T11:51:52.983294016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:43246"
ts=2026-04-27T11:51:52.989956116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:51:53.815088277Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-23755843 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-27T11:52:19.20694947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:51682"
ts=2026-04-27T11:52:22.992569158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:52:31.631813788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:56770"
level=debug ts=2026-04-27T11:52:33.935383034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:52:39.810581638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:36440"
ts=2026-04-27T11:52:49.210729224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:35114"
ts=2026-04-27T11:52:52.996319634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T11:53:22.998659013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2ddc11c1 10.130.0.73:7946"
level=debug ts=2026-04-27T11:53:25.034006678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:53:53.000905421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:54:19.216755411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:39496"
level=debug ts=2026-04-27T11:54:22.180862263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:54:23.002635019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2ddc11c1 10.130.0.73:7946"
ts=2026-04-27T11:54:31.643141682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:36246"
ts=2026-04-27T11:54:53.00486917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:55:17.76933436Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T11:55:17.770847854Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T11:55:23.007426614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2ddc11c1 10.130.0.73:7946"
ts=2026-04-27T11:55:49.224927226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:39906"
ts=2026-04-27T11:55:53.009233421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:56:01.650711061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:40840"
level=debug ts=2026-04-27T11:56:17.893295158Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:56:19.228485245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:48850"
ts=2026-04-27T11:56:23.015379168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T11:56:31.652298615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:50886"
ts=2026-04-27T11:56:49.231986549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:56098"
ts=2026-04-27T11:56:53.017859434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2ddc11c1 10.130.0.73:7946"
level=debug ts=2026-04-27T11:57:18.694434081Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:57:23.020195955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T11:57:53.022913521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2ddc11c1 10.130.0.73:7946"
level=debug ts=2026-04-27T11:58:05.128626959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:58:23.025735532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T11:58:39.071060008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T11:58:39.846071403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:53218"
level=error ts=2026-04-27T11:58:39.98169022Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T11:58:39.983189727Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T11:58:39.984694665Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T11:58:39.984721244Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T11:58:53.02840646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2ddc11c1 10.130.0.73:7946"
ts=2026-04-27T11:59:09.84941981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:49890"
level=debug ts=2026-04-27T11:59:15.576253955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T11:59:15.577780539Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T11:59:23.030275777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T11:59:49.244584156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:50830"
ts=2026-04-27T11:59:53.033091224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T11:59:54.580470769Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:00:09.854020928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:35786"
ts=2026-04-27T12:00:23.035908915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T12:00:29.656564469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:00:31.679044571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:38264"
ts=2026-04-27T12:00:49.250971248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:45574"
ts=2026-04-27T12:00:53.039002302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T12:01:09.859124127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:49942"
level=debug ts=2026-04-27T12:01:14.624303127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:01:23.042423425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T12:01:25.815471645Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-2ddc11c1 (timeout reached)"
ts=2026-04-27T12:01:28.816354936Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-2ddc11c1 has failed, no acks received"
ts=2026-04-27T12:01:49.255925153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:37524"
ts=2026-04-27T12:01:53.045522587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T12:01:54.536701912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:60542"
ts=2026-04-27T12:01:54.54955652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:60558"
level=debug ts=2026-04-27T12:02:02.256140315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:02:23.048790952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T12:02:49.260496383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:38466"
ts=2026-04-27T12:02:53.052128368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T12:03:02.2699258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T12:03:02.27148374Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T12:03:09.868206972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:52200"
ts=2026-04-27T12:03:19.263708398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:49060"
ts=2026-04-27T12:03:23.05508397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T12:03:50.267008717Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:03:53.058598088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T12:04:11.892339848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:36648"
ts=2026-04-27T12:04:19.268620308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:34172"
ts=2026-04-27T12:04:23.062367289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T12:04:35.764610093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:04:39.87424304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:33190"
ts=2026-04-27T12:04:41.894124412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:51370"
ts=2026-04-27T12:04:53.065837047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T12:05:11.896102155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:39002"
ts=2026-04-27T12:05:19.272754349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:34538"
level=debug ts=2026-04-27T12:05:19.604651313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:05:23.069341025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T12:05:39.881500894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:46400"
ts=2026-04-27T12:05:41.897802337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:34626"
ts=2026-04-27T12:05:53.07289888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T12:05:55.25826517Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:06:11.898877283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:47060"
ts=2026-04-27T12:06:23.075311964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e0d200d8 10.130.0.73:7946"
level=debug ts=2026-04-27T12:06:32.964030899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T12:06:32.965912266Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T12:06:41.900153095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:46878"
ts=2026-04-27T12:06:49.282377717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:50778"
ts=2026-04-27T12:06:53.077358372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T12:07:09.890576362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:54238"
level=debug ts=2026-04-27T12:07:12.724649067Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:07:19.285903985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:48304"
ts=2026-04-27T12:07:23.080948619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T12:07:39.893767414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:53690"
ts=2026-04-27T12:07:41.905205195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:56474"
ts=2026-04-27T12:07:53.084105294Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T12:08:01.658145839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:08:09.897606079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:60466"
ts=2026-04-27T12:08:11.906401415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:59978"
ts=2026-04-27T12:08:23.087607615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=error ts=2026-04-27T12:08:39.982701609Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T12:08:39.984653397Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T12:08:39.986502928Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T12:08:39.986529657Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T12:08:53.091263376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T12:08:59.671298035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:09:23.094540022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T12:09:38.008386918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:09:39.905598285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:41884"
ts=2026-04-27T12:09:53.096943945Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-e0d200d8 failed: dial tcp 10.130.0.73:7946: connect: connection refused"
ts=2026-04-27T12:10:05.815021223Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-e0d200d8 (timeout reached)"
ts=2026-04-27T12:10:08.815185096Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-e0d200d8 has failed, no acks received"
ts=2026-04-27T12:10:17.838053069Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-e0d200d8 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-27T12:10:19.30007606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:45878"
ts=2026-04-27T12:10:23.098796889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T12:10:37.518855461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T12:10:37.520738243Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T12:10:39.910685149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:36328"
ts=2026-04-27T12:10:49.30346965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:47134"
ts=2026-04-27T12:10:53.101891003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T12:11:09.914057466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:45144"
ts=2026-04-27T12:11:19.979490577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:38918"
ts=2026-04-27T12:11:19.994242637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:38928"
ts=2026-04-27T12:11:23.104879554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T12:11:30.616701864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:11:39.917203902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:45204"
ts=2026-04-27T12:11:49.309062068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:34592"
ts=2026-04-27T12:11:53.108673749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T12:12:23.111448924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-728ebfc5 10.130.0.73:7946"
level=debug ts=2026-04-27T12:12:25.30201406Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:12:53.112458565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-728ebfc5 10.130.0.73:7946"
level=debug ts=2026-04-27T12:13:12.218325665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:13:19.316831134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:50954"
ts=2026-04-27T12:13:23.115423332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T12:13:46.886508762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:13:53.119302961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T12:14:23.121785808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-728ebfc5 10.130.0.73:7946"
ts=2026-04-27T12:14:26.088637012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:56132"
level=debug ts=2026-04-27T12:14:31.766785397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T12:14:31.768537859Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T12:14:39.934616503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:57114"
ts=2026-04-27T12:14:53.124107196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T12:15:09.938088644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:39834"
level=debug ts=2026-04-27T12:15:13.970382093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:15:19.327943807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:35140"
ts=2026-04-27T12:15:23.126328147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-728ebfc5 10.130.0.73:7946"
ts=2026-04-27T12:15:53.128732772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T12:16:08.753096347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:16:23.131694717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T12:16:50.0914977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:16:53.1349322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T12:17:23.137068016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-728ebfc5 10.130.0.73:7946"
level=debug ts=2026-04-27T12:17:27.079969987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:17:53.139640846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T12:18:17.230168518Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T12:18:17.231709127Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T12:18:23.142711849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T12:18:26.113362792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:59366"
ts=2026-04-27T12:18:39.955625363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:59428"
level=error ts=2026-04-27T12:18:39.982320437Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T12:18:39.984259784Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T12:18:39.98583291Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T12:18:39.9858584Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T12:18:53.146088881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T12:18:56.114816531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:33222"
level=debug ts=2026-04-27T12:18:57.197909521Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:19:19.35232169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:40532"
ts=2026-04-27T12:19:23.149591146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T12:19:50.320850237Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:19:53.153244622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T12:19:56.120161938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:55058"
ts=2026-04-27T12:20:23.156451564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T12:20:26.12115834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:42362"
level=debug ts=2026-04-27T12:20:34.016808374Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:20:39.968135955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:44172"
ts=2026-04-27T12:20:53.160157899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T12:21:00.815693254Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-728ebfc5 (timeout reached)"
ts=2026-04-27T12:21:03.815018508Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-728ebfc5 has failed, no acks received"
ts=2026-04-27T12:21:05.8160343Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-728ebfc5 (timeout reached)"
ts=2026-04-27T12:21:08.815479306Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-728ebfc5 has failed, no acks received"
ts=2026-04-27T12:21:19.361924369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:41836"
ts=2026-04-27T12:21:23.16387295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T12:21:26.189052931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:21:27.004228586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:35042"
ts=2026-04-27T12:21:27.016288947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:35050"
ts=2026-04-27T12:21:49.36571522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:47070"
ts=2026-04-27T12:21:53.167134322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T12:22:02.463103524Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T12:22:02.464765109Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T12:22:19.369806263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:53680"
ts=2026-04-27T12:22:23.170538329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T12:22:40.236998209Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:22:53.174228739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T12:23:19.376577592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:55898"
ts=2026-04-27T12:23:23.178082393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
level=debug ts=2026-04-27T12:23:29.428905538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:23:53.180701116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f41e8a9 10.130.0.73:7946"
ts=2026-04-27T12:23:55.834244681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:53016"
ts=2026-04-27T12:24:09.984604147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:37934"
level=debug ts=2026-04-27T12:24:17.993677587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:24:23.183591626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T12:24:49.382810048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:53220"
ts=2026-04-27T12:24:53.185698733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f41e8a9 10.130.0.73:7946"
level=debug ts=2026-04-27T12:25:02.550380536Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:25:23.18831672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T12:25:37.980321719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T12:25:37.982474308Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T12:25:53.192310325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T12:26:09.993477906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:36050"
ts=2026-04-27T12:26:23.196135456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
ts=2026-04-27T12:26:25.849511588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:48464"
level=debug ts=2026-04-27T12:26:28.555601956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:26:39.996728731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:57850"
ts=2026-04-27T12:26:49.392737793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:58478"
ts=2026-04-27T12:26:53.198722196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f41e8a9 10.130.0.73:7946"
level=debug ts=2026-04-27T12:27:03.895879139Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:27:19.396312569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:48314"
ts=2026-04-27T12:27:23.200619361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5bfsw-e6f03928 10.129.0.95:7946"
level=debug ts=2026-04-27T12:27:40.332716098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T12:27:53.203593789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-cf9kg-d8b801fe 10.129.0.96:7946"
ts=2026-04-27T12:28:10.002904863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.95:59734"