level=info ts=2026-04-17T23:24:10.629775051Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-17T23:24:10.629796749Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-17T23:24:10.63119171Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-17T23:24:10.631506151Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-17T23:24:10.63159392Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.82 inf=eth0
level=debug ts=2026-04-17T23:24:10.632158028Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-17T23:24:10.632238068Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.82 inf=eth0
level=info ts=2026-04-17T23:24:10.633191904Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-4ce8fe01
ts=2026-04-17T23:24:10.633279937Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-17T23:24:10.633297102Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.82 advertisePort=7946
level=debug ts=2026-04-17T23:24:10.633314245Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.82 advertisePort=7946
level=info ts=2026-04-17T23:24:10.633460162Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-17T23:24:10.642345654Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-17T23:24:10.642367424Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-17T23:24:10.642381262Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=8.922943ms
level=error ts=2026-04-17T23:24:10.642390184Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-17T23:24:10.642403121Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-17T23:24:10.646099184Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T23:24:10.646129057Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-17T23:24:10.650282299Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-17T23:24:10.650397218Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-17T23:24:10.650480533Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-17T23:24:10.650570311Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-17T23:24:10.650585908Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-04-17T23:24:10.650608147Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-17T23:24:10.650877663Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-17T23:24:10.650888459Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-17T23:24:10.650894624Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-17T23:24:10.650899624Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-17T23:24:10.650948131Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-17T23:24:10.651094809Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-17T23:24:10.651114829Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-17T23:24:10.651275318Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-17T23:24:10.651282653Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-17T23:24:10.651296949Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-17T23:24:11.65224707Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-17T23:24:11.758514418Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-17T23:24:11.758580738Z caller=loki.go:599 msg="Loki started" startup_time=1.21185716s
ts=2026-04-17T23:24:12.218474021Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T23:24:12.218497195Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T23:24:15.767245713Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T23:24:15.767268994Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-17T23:24:16.759446035Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-17T23:24:16.759495497Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-17T23:24:21.236296189Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T23:24:21.236318874Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-17T23:24:28.058057187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T23:24:33.30394846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:42048"
ts=2026-04-17T23:24:35.090947018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:32870"
ts=2026-04-17T23:24:36.761844855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:50544"
ts=2026-04-17T23:24:36.933357414Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.69:7946: dial tcp 10.129.0.69:7946: connect: connection refused"
ts=2026-04-17T23:24:36.93452378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.83:7946"
ts=2026-04-17T23:24:36.936010738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.82:7946"
ts=2026-04-17T23:24:36.936031827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:40252"
ts=2026-04-17T23:24:36.936489775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.77:7946"
ts=2026-04-17T23:24:36.937729232Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.78:7946: dial tcp 10.130.0.78:7946: connect: connection refused"
ts=2026-04-17T23:24:36.938428608Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.79:7946: dial tcp 10.130.0.79:7946: connect: connection refused"
level=info ts=2026-04-17T23:24:36.938441348Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=26.296031147s
level=debug ts=2026-04-17T23:24:42.145971408Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T23:24:46.816183313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:56144"
ts=2026-04-17T23:24:50.260033888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
level=debug ts=2026-04-17T23:24:59.05006341Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T23:25:00.173957766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:55210"
ts=2026-04-17T23:25:20.263004252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
level=debug ts=2026-04-17T23:25:23.398215738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T23:25:46.81961768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:41590"
ts=2026-04-17T23:25:50.265625471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
level=debug ts=2026-04-17T23:25:53.25864242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-17T23:25:53.260501683Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T23:26:16.820871177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:33026"
ts=2026-04-17T23:26:20.267607498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
level=debug ts=2026-04-17T23:26:24.487736038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T23:26:46.822429431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:38700"
ts=2026-04-17T23:26:50.26940404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-17T23:27:00.181219069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:58118"
ts=2026-04-17T23:27:20.272048145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
level=debug ts=2026-04-17T23:27:28.137621064Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T23:27:46.825388408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:33454"
ts=2026-04-17T23:27:50.27402056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:27:54.043444791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36586"
ts=2026-04-17T23:28:20.274834874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:28:30.187096097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:44286"
level=debug ts=2026-04-17T23:28:46.744306514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T23:28:50.276942772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
ts=2026-04-17T23:29:16.831571505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:50156"
ts=2026-04-17T23:29:20.27941802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:29:24.052985658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:50540"
ts=2026-04-17T23:29:50.281162559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
level=debug ts=2026-04-17T23:30:00.96862394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T23:30:20.282513807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-17T23:30:30.195864906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:54064"
ts=2026-04-17T23:30:50.284719074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
ts=2026-04-17T23:30:54.062723162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:54062"
level=debug ts=2026-04-17T23:31:08.932092694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-17T23:31:08.933631538Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T23:31:20.287207069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-17T23:31:30.199673907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:54726"
ts=2026-04-17T23:31:50.289489613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
level=debug ts=2026-04-17T23:32:00.30844177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:32:20.291318851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:32:46.848951543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:38100"
ts=2026-04-17T23:32:50.29265936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
level=debug ts=2026-04-17T23:32:53.82746444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:33:16.850352423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:35884"
ts=2026-04-17T23:33:20.295176451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
ts=2026-04-17T23:33:30.204797809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:42266"
level=debug ts=2026-04-17T23:33:33.298692207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:33:50.298214193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
ts=2026-04-17T23:34:00.206243229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:38820"
level=debug ts=2026-04-17T23:34:15.587299828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-17T23:34:16.760354266Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-17T23:34:16.762815407Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T23:34:16.764698977Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T23:34:16.76641002Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T23:34:16.766445448Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-17T23:34:16.766459133Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-17T23:34:16.852672276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:37052"
ts=2026-04-17T23:34:20.300178247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:34:24.085571823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:47570"
ts=2026-04-17T23:34:30.207511904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:32776"
ts=2026-04-17T23:34:50.301177726Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
level=debug ts=2026-04-17T23:35:00.061958823Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T23:35:00.063659003Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T23:35:00.209070726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:59588"
ts=2026-04-17T23:35:16.85518173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:52098"
ts=2026-04-17T23:35:20.302402954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:35:24.091645596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:60342"
level=debug ts=2026-04-17T23:35:43.605623014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:35:46.856916904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:40410"
ts=2026-04-17T23:35:50.303572193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-17T23:35:54.094764776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37888"
ts=2026-04-17T23:36:00.213707965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:58358"
ts=2026-04-17T23:36:20.305530421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
level=debug ts=2026-04-17T23:36:34.613349356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:36:50.306928417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:36:54.101995226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:39608"
ts=2026-04-17T23:37:16.860883382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:56522"
ts=2026-04-17T23:37:20.30844364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-17T23:37:30.21964844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:37112"
level=debug ts=2026-04-17T23:37:31.914426839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:37:50.309558335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
level=debug ts=2026-04-17T23:38:15.004041282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:38:16.864723048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:45554"
ts=2026-04-17T23:38:20.310784763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:38:24.113869674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38982"
ts=2026-04-17T23:38:50.31241332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
level=debug ts=2026-04-17T23:38:58.64559503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T23:38:58.647784579Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T23:39:20.314481984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:39:24.121670873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:46368"
level=debug ts=2026-04-17T23:39:42.973240544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:39:50.317195852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-17T23:39:54.125321183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:40842"
ts=2026-04-17T23:40:00.235083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:48818"
ts=2026-04-17T23:40:20.319947885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
ts=2026-04-17T23:40:30.236623184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:52146"
level=debug ts=2026-04-17T23:40:42.129454469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:40:50.321973191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-17T23:41:00.238080811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:56336"
ts=2026-04-17T23:41:16.880215111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:54996"
level=debug ts=2026-04-17T23:41:17.696849989Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:41:20.324007497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-17T23:41:50.325934886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
level=debug ts=2026-04-17T23:42:02.68285947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:42:20.327906844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-17T23:42:50.328910941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-17T23:42:54.143291505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:42462"
ts=2026-04-17T23:43:00.245765737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:60028"
level=debug ts=2026-04-17T23:43:03.441466591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T23:43:03.443183997Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T23:43:20.330009814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:43:30.246792722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:48572"
level=debug ts=2026-04-17T23:43:47.762867076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:43:50.332664786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
level=error ts=2026-04-17T23:44:16.770387052Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T23:44:16.772703752Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T23:44:16.774681794Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T23:44:16.774715139Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T23:44:16.890734058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:38950"
ts=2026-04-17T23:44:20.336410895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
level=debug ts=2026-04-17T23:44:41.088998061Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:44:46.89257633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:57278"
ts=2026-04-17T23:44:50.338781542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-17T23:45:00.250682992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:60134"
ts=2026-04-17T23:45:16.893883455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:45104"
ts=2026-04-17T23:45:20.339868078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
level=debug ts=2026-04-17T23:45:41.839412918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:45:46.894787003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:60126"
ts=2026-04-17T23:45:50.341777771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
ts=2026-04-17T23:46:20.343611442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:46:24.166646425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:40432"
level=debug ts=2026-04-17T23:46:37.003204341Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:46:46.897826108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:40890"
ts=2026-04-17T23:46:50.344849661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:46:54.17024878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:58026"
ts=2026-04-17T23:47:00.260071173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:43316"
ts=2026-04-17T23:47:16.898686103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:45210"
ts=2026-04-17T23:47:20.346591456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
level=debug ts=2026-04-17T23:47:31.343607442Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T23:47:31.345338119Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T23:47:50.348129343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
level=debug ts=2026-04-17T23:48:14.342444538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:48:20.349911506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
ts=2026-04-17T23:48:24.178821432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:43400"
ts=2026-04-17T23:48:50.352512373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
level=debug ts=2026-04-17T23:48:50.609817715Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:48:54.181892448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:47860"
ts=2026-04-17T23:49:00.269172601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:59714"
ts=2026-04-17T23:49:16.908208917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:33316"
ts=2026-04-17T23:49:20.353591535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
level=debug ts=2026-04-17T23:49:45.80758211Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:49:50.355725167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
ts=2026-04-17T23:50:20.357786455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:50:30.274908219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:37592"
level=debug ts=2026-04-17T23:50:41.7297434Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:50:46.911723802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:45160"
ts=2026-04-17T23:50:50.359447716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:51:16.913102685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:41476"
ts=2026-04-17T23:51:20.361592281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
level=debug ts=2026-04-17T23:51:41.052066608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T23:51:41.053833437Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T23:51:50.363489281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:52:00.281700637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:36810"
ts=2026-04-17T23:52:16.915822882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:45756"
ts=2026-04-17T23:52:20.364986579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
level=debug ts=2026-04-17T23:52:23.30370228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:52:50.367567501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
level=debug ts=2026-04-17T23:53:11.694241276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:53:20.370082308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:53:30.287610424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:45682"
ts=2026-04-17T23:53:50.371124722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:54:00.288969312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:41282"
level=debug ts=2026-04-17T23:54:08.767196845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T23:54:16.769703997Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T23:54:16.771542882Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T23:54:16.773132037Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T23:54:16.773168113Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T23:54:16.920410727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:51496"
ts=2026-04-17T23:54:20.373423459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
level=debug ts=2026-04-17T23:54:46.882243515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:54:50.375665602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-17T23:55:00.291984376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:44836"
ts=2026-04-17T23:55:16.925238989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:56314"
ts=2026-04-17T23:55:20.377733011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
ts=2026-04-17T23:55:24.224283919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:41748"
ts=2026-04-17T23:55:30.293306577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:55404"
level=debug ts=2026-04-17T23:55:41.442033741Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T23:55:41.443846589Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T23:55:50.380052191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-17T23:56:20.382628379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
ts=2026-04-17T23:56:24.230266012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:42822"
level=debug ts=2026-04-17T23:56:32.3673359Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:56:50.384629352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:57:20.385635177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:57:30.303429508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:48324"
level=debug ts=2026-04-17T23:57:32.340729003Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:57:50.387492797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
ts=2026-04-17T23:57:54.239065139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:57130"
ts=2026-04-17T23:58:20.389660641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
level=debug ts=2026-04-17T23:58:32.987888286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:58:46.945438576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:38858"
ts=2026-04-17T23:58:50.391374524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-17T23:58:54.245385327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:42970"
ts=2026-04-17T23:59:16.947312204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:41466"
level=debug ts=2026-04-17T23:59:19.230130723Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T23:59:20.39231126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-17T23:59:50.393604467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
level=debug ts=2026-04-17T23:59:56.872903297Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T23:59:56.874698802Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-18T00:00:00.312975928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:34128"
ts=2026-04-18T00:00:20.394804018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-18T00:00:30.32730603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:39492"
level=debug ts=2026-04-18T00:00:33.783554354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T00:00:50.396423484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-18T00:00:54.258401012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:45232"
level=debug ts=2026-04-18T00:01:12.029232008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T00:01:20.397609626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-18T00:01:46.959726304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:49714"
ts=2026-04-18T00:01:50.399882176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
level=debug ts=2026-04-18T00:02:10.26619591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T00:02:20.405359004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
ts=2026-04-18T00:02:46.962952491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:39590"
ts=2026-04-18T00:02:50.410180924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-18T00:03:00.337629363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:48996"
level=debug ts=2026-04-18T00:03:03.67779539Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T00:03:16.963980913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:53132"
ts=2026-04-18T00:03:20.412925804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
level=debug ts=2026-04-18T00:03:45.546209635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-18T00:03:45.547786881Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-18T00:03:50.414845766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-18T00:03:54.278919604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:39342"
ts=2026-04-18T00:04:00.342890196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:32770"
level=error ts=2026-04-18T00:04:16.771380723Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-18T00:04:16.773159996Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-18T00:04:16.774945294Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-18T00:04:16.774970553Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-18T00:04:16.966992444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:41070"
ts=2026-04-18T00:04:20.416404556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
level=debug ts=2026-04-18T00:04:25.856689945Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T00:04:30.344466969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:40366"
ts=2026-04-18T00:04:50.419155716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
ts=2026-04-18T00:05:00.345475269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:51326"
level=debug ts=2026-04-18T00:05:04.679915468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T00:05:20.421638922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-18T00:05:50.424321737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
level=debug ts=2026-04-18T00:06:00.713572823Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T00:06:16.974330607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:33886"
ts=2026-04-18T00:06:20.426686109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-18T00:06:24.294361515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:47740"
level=debug ts=2026-04-18T00:06:36.053525322Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T00:06:50.428114841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-000bd48e 10.130.0.83:7946"
ts=2026-04-18T00:06:54.29792228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:43212"
level=debug ts=2026-04-18T00:07:14.633806393Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-18T00:07:14.635342067Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-18T00:07:20.429927321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-18T00:07:46.980689798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:57338"
ts=2026-04-18T00:07:50.432718998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
level=debug ts=2026-04-18T00:07:54.120542967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T00:08:20.435349372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
level=debug ts=2026-04-18T00:08:31.867136695Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T00:08:46.983080606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:48028"
ts=2026-04-18T00:08:50.438953388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dldgv-efbafa5f 10.128.0.83:7946"
ts=2026-04-18T00:09:16.984027409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:50898"
level=debug ts=2026-04-18T00:09:18.822074012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T00:09:20.440853575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-18T00:09:24.314059092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37118"
ts=2026-04-18T00:09:30.362471242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:56782"
ts=2026-04-18T00:09:50.442464614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l9bjs-799b8dfa 10.130.0.77:7946"
ts=2026-04-18T00:09:54.317317437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36152"
level=debug ts=2026-04-18T00:10:06.380175529Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"