level=info ts=2026-04-21T10:59:20.52131572Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-21T10:59:20.521596304Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-21T10:59:20.522974779Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-21T10:59:20.523374503Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T10:59:20.52347605Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.82 inf=eth0
level=debug ts=2026-04-21T10:59:20.524307742Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T10:59:20.524392559Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.82 inf=eth0
level=info ts=2026-04-21T10:59:20.524437798Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-19ec21e9
ts=2026-04-21T10:59:20.524510964Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-21T10:59:20.524525753Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.82 advertisePort=7946
level=debug ts=2026-04-21T10:59:20.524543667Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.82 advertisePort=7946
level=info ts=2026-04-21T10:59:20.524694861Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-21T10:59:20.531800839Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T10:59:20.531818105Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T10:59:20.531828363Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=7.135434ms
level=error ts=2026-04-21T10:59:20.531837351Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-21T10:59:20.531849263Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-21T10:59:20.534220483Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T10:59:20.534249167Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-21T10:59:20.542389616Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-21T10:59:20.5424387Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-21T10:59:20.542453758Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-21T10:59:20.542470658Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-21T10:59:20.542546718Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-21T10:59:20.542626502Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-21T10:59:20.542628982Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-21T10:59:20.542640211Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-21T10:59:20.54267635Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-21T10:59:20.542688804Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-21T10:59:20.542716676Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-21T10:59:20.542763889Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-21T10:59:20.542788778Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-21T10:59:20.542941455Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-21T10:59:20.542981154Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-21T10:59:20.543050548Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-21T10:59:21.543938727Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
ts=2026-04-21T10:59:21.605620746Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T10:59:21.605651681Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T10:59:21.723018492Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-21T10:59:21.723070561Z caller=loki.go:599 msg="Loki started" startup_time=1.290767791s
ts=2026-04-21T10:59:25.308278572Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T10:59:25.308302369Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T10:59:26.723206892Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-21T10:59:26.723257894Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-21T10:59:31.745244658Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T10:59:31.745270289Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-21T10:59:39.35537134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T10:59:45.711909421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.82:7946"
ts=2026-04-21T10:59:45.712059001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:47006"
ts=2026-04-21T10:59:45.713276763Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.76:7946: dial tcp 10.130.0.76:7946: connect: connection refused"
ts=2026-04-21T10:59:45.714125873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.74:7946"
ts=2026-04-21T10:59:45.715315161Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.79:7946: dial tcp 10.130.0.79:7946: connect: connection refused"
ts=2026-04-21T10:59:45.715955726Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.80:7946"
ts=2026-04-21T10:59:45.717211337Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.83:7946: dial tcp 10.130.0.83:7946: connect: connection refused"
level=info ts=2026-04-21T10:59:45.717221447Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=25.185366441s
ts=2026-04-21T10:59:46.47594601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:48452"
ts=2026-04-21T10:59:46.496909857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:45876"
ts=2026-04-21T10:59:51.546995249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:54932"
ts=2026-04-21T10:59:57.434554332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T10:59:57.740169502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T11:00:01.388474761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:45862"
level=debug ts=2026-04-21T11:00:19.342443396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T11:00:27.436141925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:00:32.626146067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:49258"
level=debug ts=2026-04-21T11:00:38.815925132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T11:00:51.549401152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:58074"
ts=2026-04-21T11:00:57.437772085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:00:59.0107868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:41192"
level=debug ts=2026-04-21T11:01:06.926269404Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T11:01:06.928368651Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:01:21.550503475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:41542"
ts=2026-04-21T11:01:27.439478703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-778d1bfe 10.130.0.81:7946"
ts=2026-04-21T11:01:32.628564937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:55360"
level=debug ts=2026-04-21T11:01:45.56913747Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T11:01:57.441202549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-778d1bfe 10.130.0.81:7946"
ts=2026-04-21T11:02:21.553333623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:52994"
ts=2026-04-21T11:02:27.442707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-778d1bfe 10.130.0.81:7946"
level=debug ts=2026-04-21T11:02:45.508155928Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T11:02:57.443936412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:02:59.01675452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:59026"
ts=2026-04-21T11:03:27.445039102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:03:57.446638359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T11:03:59.343634468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T11:04:27.447677946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:04:32.636389182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:33262"
level=debug ts=2026-04-21T11:04:53.740638703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T11:04:57.449331196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-778d1bfe 10.130.0.81:7946"
ts=2026-04-21T11:04:59.021593415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:36602"
ts=2026-04-21T11:05:21.561983715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:55704"
ts=2026-04-21T11:05:27.450938252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:05:32.639074067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:42734"
level=debug ts=2026-04-21T11:05:50.999715056Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T11:05:51.001369932Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:05:51.563300386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:49988"
ts=2026-04-21T11:05:57.451984007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:06:21.564772007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:41068"
ts=2026-04-21T11:06:27.453639528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:06:29.026302156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:37460"
level=debug ts=2026-04-21T11:06:35.096839202Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:06:57.454396981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:07:27.45539141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:07:32.643120561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:36756"
level=debug ts=2026-04-21T11:07:34.282099552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:07:57.456629217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:07:59.030896683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:35410"
ts=2026-04-21T11:08:21.571125716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:41408"
level=debug ts=2026-04-21T11:08:21.593537433Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:08:27.457984108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:08:32.6460507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:38752"
ts=2026-04-21T11:08:57.459483353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-778d1bfe 10.130.0.81:7946"
ts=2026-04-21T11:08:59.033620013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:59382"
level=debug ts=2026-04-21T11:09:16.028724884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:09:21.573254742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:55292"
level=info ts=2026-04-21T11:09:26.724087127Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-21T11:09:26.727385309Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:09:26.729175844Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:09:26.730923037Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:09:26.730950849Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T11:09:26.730963819Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-21T11:09:27.461042801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:09:32.649069231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:34028"
level=debug ts=2026-04-21T11:09:54.130691619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T11:09:54.132649416Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:09:57.462663903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:09:59.036323722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:39634"
ts=2026-04-21T11:10:21.575410664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:46294"
ts=2026-04-21T11:10:27.464238089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:10:32.651692046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:41628"
level=debug ts=2026-04-21T11:10:44.981175418Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:10:57.464985733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:11:27.46591661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:11:29.039894484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56534"
ts=2026-04-21T11:11:32.65425947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:46604"
level=debug ts=2026-04-21T11:11:34.395240501Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:11:51.579350934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:38688"
ts=2026-04-21T11:11:57.467151114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:12:02.6550544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:45500"
ts=2026-04-21T11:12:27.468119158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:12:29.042682881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:34696"
level=debug ts=2026-04-21T11:12:31.225285315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:12:57.469630059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-778d1bfe 10.130.0.81:7946"
level=debug ts=2026-04-21T11:13:24.062193601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:13:27.470710324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-778d1bfe 10.130.0.81:7946"
ts=2026-04-21T11:13:29.045466428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:40562"
ts=2026-04-21T11:13:51.585176459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:35154"
ts=2026-04-21T11:13:57.471992585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-778d1bfe 10.130.0.81:7946"
level=debug ts=2026-04-21T11:13:59.420925124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T11:13:59.422457763Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:14:27.473264385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:14:32.660468756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:54396"
level=debug ts=2026-04-21T11:14:38.730868941Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:14:57.474315243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:14:59.048487264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:41176"
ts=2026-04-21T11:15:27.475259329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:15:29.049308573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:51766"
level=debug ts=2026-04-21T11:15:35.115999238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:15:51.590212682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:38804"
ts=2026-04-21T11:15:57.477158252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:15:59.050161661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:59132"
level=debug ts=2026-04-21T11:16:12.875212142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:16:21.591902578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:37548"
ts=2026-04-21T11:16:27.478161999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:16:32.666679241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:33358"
ts=2026-04-21T11:16:57.479015532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:17:02.667762697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:51490"
level=debug ts=2026-04-21T11:17:04.778473926Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:17:27.480245243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=debug ts=2026-04-21T11:17:40.249348818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T11:17:40.251363151Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:17:57.481080171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T11:18:26.966450629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:18:27.48196794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:18:29.057046509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:37624"
ts=2026-04-21T11:18:51.598728179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:52914"
ts=2026-04-21T11:18:57.483205567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:19:21.600459407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:52962"
level=debug ts=2026-04-21T11:19:23.824814726Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T11:19:26.733700102Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:19:26.735584824Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:19:26.737471933Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:19:26.737498578Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:19:27.484725178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:19:57.487147758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-778d1bfe 10.130.0.81:7946"
ts=2026-04-21T11:20:02.682999364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:40600"
level=debug ts=2026-04-21T11:20:24.010872158Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:20:27.49255514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-778d1bfe 10.130.0.81:7946"
ts=2026-04-21T11:20:29.072469981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48822"
ts=2026-04-21T11:20:57.493831441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-778d1bfe 10.130.0.81:7946"
level=debug ts=2026-04-21T11:20:58.476554158Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:21:02.689806932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:40988"
ts=2026-04-21T11:21:21.607102683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:48978"
ts=2026-04-21T11:21:27.495038188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-778d1bfe 10.130.0.81:7946"
ts=2026-04-21T11:21:29.076099827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:37734"
level=debug ts=2026-04-21T11:21:35.624912467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T11:21:35.626634776Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:21:57.496887644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:22:02.691977013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:41760"
level=debug ts=2026-04-21T11:22:16.92654116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:22:27.498806477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:22:29.079081066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:53182"
ts=2026-04-21T11:22:51.611434729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:38676"
ts=2026-04-21T11:22:57.500156046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T11:22:59.32715006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:23:27.501915563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-778d1bfe 10.130.0.81:7946"
ts=2026-04-21T11:23:29.081676655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:37234"
level=debug ts=2026-04-21T11:23:49.037988713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:23:51.614332164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:54828"
ts=2026-04-21T11:23:57.503168994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:23:59.083163563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:60528"
ts=2026-04-21T11:24:02.697315615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:35878"
level=debug ts=2026-04-21T11:24:23.766721279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:24:27.504392309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:24:51.616946377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:53730"
ts=2026-04-21T11:24:57.5052466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=debug ts=2026-04-21T11:25:02.342440406Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T11:25:02.34409084Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:25:27.50648946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:25:29.086265478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48806"
level=debug ts=2026-04-21T11:25:38.832210398Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:25:51.61963416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:52522"
ts=2026-04-21T11:25:57.507575209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:26:27.509152938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T11:26:35.167378704Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:26:57.511025938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:27:21.624861295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:60056"
ts=2026-04-21T11:27:27.512135379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-778d1bfe 10.130.0.81:7946"
level=debug ts=2026-04-21T11:27:29.683299942Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:27:32.707105283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:45566"
ts=2026-04-21T11:27:51.626602825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:36806"
ts=2026-04-21T11:27:57.513154683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:28:02.709051581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:58140"
ts=2026-04-21T11:28:27.51484215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-778d1bfe 10.130.0.81:7946"
level=debug ts=2026-04-21T11:28:28.650284534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:28:32.710984091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:50114"
ts=2026-04-21T11:28:51.629997507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:50486"
ts=2026-04-21T11:28:57.516094336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=debug ts=2026-04-21T11:29:21.455392595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T11:29:21.457472246Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:29:21.630820014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:51838"
level=error ts=2026-04-21T11:29:26.734109438Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:29:26.735917746Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:29:26.737530059Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:29:26.737554086Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:29:27.517797668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:29:57.518982328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T11:30:11.612618112Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:30:13.435784668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:58450"
ts=2026-04-21T11:30:13.441659027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:58452"
ts=2026-04-21T11:30:21.633596364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:55002"
ts=2026-04-21T11:30:27.520710367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c71f760 10.130.0.81:7946"
ts=2026-04-21T11:30:47.525272086Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-778d1bfe (timeout reached)"
ts=2026-04-21T11:30:50.526078953Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-778d1bfe as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-21T11:30:50.52612181Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-778d1bfe has failed, no acks received"
ts=2026-04-21T11:30:57.522310425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=debug ts=2026-04-21T11:31:05.758711528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:31:21.636638233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:40768"
ts=2026-04-21T11:31:27.52393732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:31:51.638524959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:43482"
level=debug ts=2026-04-21T11:31:54.872864829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:31:57.525831174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c71f760 10.130.0.81:7946"
ts=2026-04-21T11:32:02.722071082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:35330"
ts=2026-04-21T11:32:21.640363682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:34306"
ts=2026-04-21T11:32:27.527652214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=debug ts=2026-04-21T11:32:32.575942383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:32:32.723218212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:50494"
ts=2026-04-21T11:32:51.641784222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:45194"
ts=2026-04-21T11:32:57.529554128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=debug ts=2026-04-21T11:33:24.146681276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T11:33:24.148375313Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:33:27.530432937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:33:41.626204476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:57174"
ts=2026-04-21T11:33:57.531317886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c71f760 10.130.0.81:7946"
level=debug ts=2026-04-21T11:34:25.770879739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:34:27.532206885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:34:41.63020062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:50272"
ts=2026-04-21T11:34:51.647982553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:60380"
ts=2026-04-21T11:34:57.533355119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=debug ts=2026-04-21T11:35:01.072544368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:35:11.631898318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:43588"
ts=2026-04-21T11:35:21.649868441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:37988"
ts=2026-04-21T11:35:27.534581593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=debug ts=2026-04-21T11:35:37.010892065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:35:57.53559785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:36:11.635021569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:58906"
level=debug ts=2026-04-21T11:36:14.852033271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:36:27.537059511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:36:51.662258369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:54898"
level=debug ts=2026-04-21T11:36:55.955990039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T11:36:55.959070193Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:36:57.538046101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:37:11.639137336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:40426"
ts=2026-04-21T11:37:27.539407383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c71f760 10.130.0.81:7946"
level=debug ts=2026-04-21T11:37:48.372294088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:37:57.540610645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:38:11.641936735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:47050"
ts=2026-04-21T11:38:27.54178497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c71f760 10.130.0.81:7946"
ts=2026-04-21T11:38:32.740554998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:55530"
level=debug ts=2026-04-21T11:38:47.961756785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:38:57.543484948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:39:11.645049781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:55046"
level=debug ts=2026-04-21T11:39:26.14944986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T11:39:26.733236394Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:39:26.734957509Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:39:26.736599241Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:39:26.73662473Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:39:27.544988696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:39:51.675190541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:49262"
ts=2026-04-21T11:39:57.546031608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=debug ts=2026-04-21T11:40:05.305121973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:40:21.67716958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:46302"
ts=2026-04-21T11:40:27.547836656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c71f760 10.130.0.81:7946"
ts=2026-04-21T11:40:44.336093784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:37348"
ts=2026-04-21T11:40:44.343099953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:37356"
ts=2026-04-21T11:40:52.525578942Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8c71f760 (timeout reached)"
ts=2026-04-21T11:40:55.525504898Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8c71f760 has failed, no acks received"
ts=2026-04-21T11:40:57.548722269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=debug ts=2026-04-21T11:41:00.240687625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T11:41:00.24231445Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:41:10.453112111Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-8c71f760 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-21T11:41:21.680403977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:41036"
ts=2026-04-21T11:41:27.549542627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:41:34.374155118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:49154"
level=debug ts=2026-04-21T11:41:42.530148044Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:41:51.681684862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:39092"
ts=2026-04-21T11:41:57.550521365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:42:02.75041605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:35666"
ts=2026-04-21T11:42:04.375295506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:49262"
ts=2026-04-21T11:42:27.551594702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:42:32.75171495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:51624"
ts=2026-04-21T11:42:34.376738284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:58916"
level=debug ts=2026-04-21T11:42:35.863153685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:42:51.684730587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:45018"
ts=2026-04-21T11:42:57.553218098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:43:04.378321981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:33320"
level=debug ts=2026-04-21T11:43:15.296150991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:43:27.554430329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:43:34.379376401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:35434"
ts=2026-04-21T11:43:57.556001611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:44:04.381366146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48458"
level=debug ts=2026-04-21T11:44:14.147815786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:44:27.557727449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:44:51.690225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:43456"
level=debug ts=2026-04-21T11:44:56.666228407Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T11:44:56.668175903Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:44:57.559143955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:45:02.75957545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:35998"
ts=2026-04-21T11:45:27.560875016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T11:45:45.655593436Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:45:57.562391247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:46:02.763263934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:40796"
ts=2026-04-21T11:46:27.56346555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-833cd140 10.130.0.81:7946"
level=debug ts=2026-04-21T11:46:32.958843495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:46:51.695812036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:46806"
ts=2026-04-21T11:46:57.5651473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:47:27.566360885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-833cd140 10.130.0.81:7946"
level=debug ts=2026-04-21T11:47:29.858337731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:47:57.567968125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-833cd140 10.130.0.81:7946"
ts=2026-04-21T11:48:04.395627493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:42946"
level=debug ts=2026-04-21T11:48:12.615726075Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:48:27.569312592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:48:42.525199761Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-833cd140 (timeout reached)"
ts=2026-04-21T11:48:45.525406359Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-833cd140 has failed, no acks received"
ts=2026-04-21T11:48:45.712360709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:36678"
ts=2026-04-21T11:48:45.719132864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:36686"
ts=2026-04-21T11:48:51.701329789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:45564"
ts=2026-04-21T11:48:52.525475979Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-833cd140 (timeout reached)"
ts=2026-04-21T11:48:55.523596538Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-833cd140 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-21T11:48:55.525721748Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-833cd140 has failed, no acks received"
ts=2026-04-21T11:48:57.570546196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T11:48:59.880601742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T11:48:59.882173141Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:49:26.734520151Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:49:26.736804726Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:49:26.738452977Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:49:26.738478985Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:49:27.571758535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2ee9a29 10.130.0.81:7946"
level=debug ts=2026-04-21T11:49:46.839359678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:49:57.573312084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:50:21.704356287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:38560"
level=debug ts=2026-04-21T11:50:24.095773233Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:50:27.574343634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:50:32.777027969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:36812"
ts=2026-04-21T11:50:57.575279789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2ee9a29 10.130.0.81:7946"
level=debug ts=2026-04-21T11:51:08.66389271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:51:17.36186592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:47696"
ts=2026-04-21T11:51:27.577133518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2ee9a29 10.130.0.81:7946"
level=debug ts=2026-04-21T11:51:48.939424726Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:51:51.70825629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:50118"
ts=2026-04-21T11:51:57.578365922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2ee9a29 10.130.0.81:7946"
ts=2026-04-21T11:52:02.780398867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:58424"
ts=2026-04-21T11:52:17.364459436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:36110"
ts=2026-04-21T11:52:27.579262574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T11:52:30.448717006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T11:52:30.450286487Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:52:32.781950468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:46450"
ts=2026-04-21T11:52:57.581049975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:53:02.782764888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:50924"
level=debug ts=2026-04-21T11:53:16.364822986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:53:27.582885168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:53:32.784387749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:41282"
ts=2026-04-21T11:53:57.584093957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:54:02.785621348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:53340"
level=debug ts=2026-04-21T11:54:12.336808326Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:54:17.370722436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:60038"
ts=2026-04-21T11:54:27.585521693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:54:57.586697447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T11:55:04.160263585Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:55:27.588109811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2ee9a29 10.130.0.81:7946"
level=debug ts=2026-04-21T11:55:50.337155378Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:55:57.589131736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T11:56:21.719852126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:34096"
ts=2026-04-21T11:56:27.5904583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T11:56:47.599316533Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T11:56:47.600941721Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:56:57.59203965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2ee9a29 10.130.0.81:7946"
ts=2026-04-21T11:57:17.37792425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56150"
ts=2026-04-21T11:57:21.722521951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:39532"
ts=2026-04-21T11:57:27.593032945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=debug ts=2026-04-21T11:57:42.154857069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:57:57.593979488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:58:02.795664474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:49342"
ts=2026-04-21T11:58:17.526238167Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-c2ee9a29 (timeout reached)"
ts=2026-04-21T11:58:20.526154841Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-c2ee9a29 has failed, no acks received"
ts=2026-04-21T11:58:27.594928032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:58:32.525688199Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-c2ee9a29 (timeout reached)"
ts=2026-04-21T11:58:35.526543961Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-c2ee9a29 has failed, no acks received"
level=debug ts=2026-04-21T11:58:37.13459595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:58:37.721077812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48984"
ts=2026-04-21T11:58:37.727718829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48988"
ts=2026-04-21T11:58:40.453625283Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-c2ee9a29 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-21T11:58:51.726298861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:49866"
ts=2026-04-21T11:58:57.596099506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T11:59:02.797733458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:57076"
ts=2026-04-21T11:59:21.727218198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:58922"
level=error ts=2026-04-21T11:59:26.735203606Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:59:26.737470934Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:59:26.739423178Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T11:59:26.73945152Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T11:59:27.596836265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T11:59:39.058578912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T11:59:49.953457431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:52184"
ts=2026-04-21T11:59:57.597789719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=debug ts=2026-04-21T12:00:21.173683442Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:00:27.599021629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=debug ts=2026-04-21T12:00:55.929997905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T12:00:55.932048117Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T12:00:57.600338705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T12:01:02.804215659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:51156"
ts=2026-04-21T12:01:27.601183671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a3b1bd2b 10.130.0.81:7946"
ts=2026-04-21T12:01:32.805235378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:56704"
level=debug ts=2026-04-21T12:01:48.991322217Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:01:57.60207332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a3b1bd2b 10.130.0.81:7946"
ts=2026-04-21T12:02:21.735302814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:57084"
ts=2026-04-21T12:02:27.603626456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T12:02:29.271854196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:02:57.604707125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a3b1bd2b 10.130.0.81:7946"
level=debug ts=2026-04-21T12:03:03.945346236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:03:27.606059097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a3b1bd2b 10.130.0.81:7946"
level=debug ts=2026-04-21T12:03:48.382090496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:03:57.607305707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T12:04:19.965100265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:42112"
ts=2026-04-21T12:04:27.608492724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T12:04:38.145807918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T12:04:38.147410228Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T12:04:57.609717147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T12:05:27.611078264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T12:05:32.81574071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:52362"
level=debug ts=2026-04-21T12:05:38.55325728Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:05:57.612932031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T12:06:02.817497772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:55464"
level=debug ts=2026-04-21T12:06:21.074794476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:06:21.745145934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:39204"
ts=2026-04-21T12:06:27.614774957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T12:06:49.971444235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:44884"
ts=2026-04-21T12:06:57.616317902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a3b1bd2b 10.130.0.81:7946"
level=debug ts=2026-04-21T12:07:12.437121513Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:07:21.747328821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:43396"
ts=2026-04-21T12:07:27.617760828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T12:07:42.525702037Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-a3b1bd2b (timeout reached)"
ts=2026-04-21T12:07:45.526108446Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-a3b1bd2b has failed, no acks received"
ts=2026-04-21T12:07:57.526042793Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-a3b1bd2b (timeout reached)"
ts=2026-04-21T12:07:57.618709336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T12:08:00.526322948Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-a3b1bd2b has failed, no acks received"
ts=2026-04-21T12:08:02.823293817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:34344"
ts=2026-04-21T12:08:12.717193684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48660"
ts=2026-04-21T12:08:12.724997955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48672"
level=debug ts=2026-04-21T12:08:14.639685492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:08:21.750058145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:38344"
ts=2026-04-21T12:08:27.620328685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T12:08:32.824979375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:50284"
ts=2026-04-21T12:08:51.520921499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:59722"
ts=2026-04-21T12:08:57.621718609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T12:09:10.065301376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T12:09:10.067269828Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T12:09:26.734378619Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T12:09:26.73648558Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T12:09:26.73825786Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T12:09:26.738284221Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T12:09:27.622835029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T12:09:32.828154559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:41546"
ts=2026-04-21T12:09:57.624095052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8e06e546 10.130.0.81:7946"
ts=2026-04-21T12:10:02.829555219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:37628"
level=debug ts=2026-04-21T12:10:09.719600555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:10:27.625092355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T12:10:32.830958078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:47630"
level=debug ts=2026-04-21T12:10:57.274419892Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:10:57.626643708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8e06e546 10.130.0.81:7946"
ts=2026-04-21T12:11:27.628126549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8e06e546 10.130.0.81:7946"
ts=2026-04-21T12:11:32.833309295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:45526"
level=debug ts=2026-04-21T12:11:43.122771154Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:11:51.530371828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:35136"
ts=2026-04-21T12:11:57.629506334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T12:12:21.531530048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48400"
ts=2026-04-21T12:12:21.76190614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:33464"
ts=2026-04-21T12:12:27.631076487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8e06e546 10.130.0.81:7946"
level=debug ts=2026-04-21T12:12:32.775245412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:12:57.631958178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8e06e546 10.130.0.81:7946"
ts=2026-04-21T12:13:02.836734722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:57122"
level=debug ts=2026-04-21T12:13:14.542512232Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T12:13:14.544044941Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T12:13:27.633147909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T12:13:54.576501963Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:13:57.634137999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8e06e546 10.130.0.81:7946"
ts=2026-04-21T12:14:17.525546161Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8e06e546 (timeout reached)"
ts=2026-04-21T12:14:20.526328665Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8e06e546 has failed, no acks received"
ts=2026-04-21T12:14:27.525147735Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8e06e546 (timeout reached)"
ts=2026-04-21T12:14:27.634899921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T12:14:30.525736339Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8e06e546 has failed, no acks received"
ts=2026-04-21T12:14:32.839907371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:55694"
level=debug ts=2026-04-21T12:14:36.075012036Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:14:57.636132426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T12:15:02.84126133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:47492"
level=debug ts=2026-04-21T12:15:17.506152385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:15:21.769125906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:56292"
ts=2026-04-21T12:15:26.696363543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:38970"
ts=2026-04-21T12:15:26.703518635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:38974"
ts=2026-04-21T12:15:27.637958692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T12:15:32.842965274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:49080"
ts=2026-04-21T12:15:51.770267752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:39138"
ts=2026-04-21T12:15:57.639393651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T12:16:17.081021999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:16:27.641034289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58ea1800 10.130.0.81:7946"
ts=2026-04-21T12:16:51.772022934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:36140"
ts=2026-04-21T12:16:57.641968986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T12:17:02.84655111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:34368"
level=debug ts=2026-04-21T12:17:09.170824575Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T12:17:09.172403316Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T12:17:27.643056547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
ts=2026-04-21T12:17:32.848147561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:54474"
ts=2026-04-21T12:17:57.644250431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T12:18:01.864120509Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:18:21.77583756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:57562"
ts=2026-04-21T12:18:27.645430877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T12:18:37.456144368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:18:40.612590257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:42444"
ts=2026-04-21T12:18:57.64689053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58ea1800 10.130.0.81:7946"
level=debug ts=2026-04-21T12:19:20.331321859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T12:19:26.733728392Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T12:19:26.735514823Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T12:19:26.737270408Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T12:19:26.737301555Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T12:19:27.647939137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T12:19:57.649283993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=debug ts=2026-04-21T12:20:03.479456238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:20:27.650672861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58ea1800 10.130.0.81:7946"
level=debug ts=2026-04-21T12:20:52.16806226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T12:20:52.169691458Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T12:20:57.651570142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
ts=2026-04-21T12:21:21.78314981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:42806"
ts=2026-04-21T12:21:27.652908269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=debug ts=2026-04-21T12:21:28.163711908Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:21:32.858902929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:43810"
ts=2026-04-21T12:21:57.654352316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58ea1800 10.130.0.81:7946"
level=debug ts=2026-04-21T12:22:23.287553261Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T12:22:27.656288714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58ea1800 10.130.0.81:7946"
ts=2026-04-21T12:22:51.787892414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:56708"
ts=2026-04-21T12:22:57.657595505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=debug ts=2026-04-21T12:23:14.358286379Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"