level=info ts=2026-05-02T02:40:25.888571321Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-02T02:40:25.888658849Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-02T02:40:25.890203035Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-02T02:40:25.890615137Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-02T02:40:25.89071832Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.78 inf=eth0
level=debug ts=2026-05-02T02:40:25.891665931Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-02T02:40:25.891749004Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.78 inf=eth0
level=info ts=2026-05-02T02:40:25.891764949Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-88351b2b
ts=2026-05-02T02:40:25.891846168Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-02T02:40:25.891860092Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.78 advertisePort=7946
level=debug ts=2026-05-02T02:40:25.891879691Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.78 advertisePort=7946
level=info ts=2026-05-02T02:40:25.892040714Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-02T02:40:25.896666225Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-02T02:40:25.896680503Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-02T02:40:25.896690104Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=4.651346ms
level=error ts=2026-05-02T02:40:25.896698407Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-02T02:40:25.896710447Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-02T02:40:25.899226574Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-02T02:40:25.899243443Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-02T02:40:25.908382103Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-02T02:40:25.908428202Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-02T02:40:25.908456592Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-02T02:40:25.908479489Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-02T02:40:25.908541755Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-05-02T02:40:25.908676042Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-02T02:40:25.908873843Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-05-02T02:40:25.908889231Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-05-02T02:40:25.908947673Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-05-02T02:40:25.908966945Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-02T02:40:25.909052937Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-02T02:40:25.909039498Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-02T02:40:25.909066529Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-02T02:40:25.909254339Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-02T02:40:25.909275674Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-02T02:40:25.909296533Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-02T02:40:26.910154554Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
ts=2026-05-02T02:40:27.018117968Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-02T02:40:27.018140429Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-02T02:40:27.053246587Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-02T02:40:27.053314112Z caller=loki.go:615 msg="Loki started" startup_time=1.235294069s
ts=2026-05-02T02:40:30.140589612Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-02T02:40:30.140612323Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-02T02:40:32.054266709Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-02T02:40:32.054314745Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-02T02:40:35.499843093Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-02T02:40:35.49986975Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-02T02:40:35.66788098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-02T02:40:43.328194781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:40082"
ts=2026-05-02T02:40:44.356125362Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.76:7946: dial tcp 10.130.0.76:7946: connect: connection refused"
ts=2026-05-02T02:40:44.357034742Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.81:7946: dial tcp 10.130.0.81:7946: connect: connection refused"
ts=2026-05-02T02:40:44.357148189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.78:7946"
ts=2026-05-02T02:40:44.357244293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:54248"
ts=2026-05-02T02:40:44.35762992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.77:7946"
ts=2026-05-02T02:40:44.358058728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.75:7946"
ts=2026-05-02T02:40:44.360316298Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.80:7946: dial tcp 10.129.0.80:7946: connect: connection refused"
level=info ts=2026-05-02T02:40:44.360326965Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=18.463610534s
ts=2026-05-02T02:40:45.949492069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:55890"
ts=2026-05-02T02:40:50.418289242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:59124"
level=debug ts=2026-05-02T02:40:57.632558236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-02T02:41:07.972535957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:46820"
ts=2026-05-02T02:41:10.511498637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T02:41:16.611245578Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-02T02:41:40.323237151Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-02T02:41:40.513065922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T02:42:10.513905164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T02:42:12.869893185Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-02T02:42:12.871426398Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T02:42:40.515593986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T02:42:52.756576667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:56524"
level=debug ts=2026-05-02T02:42:52.965071982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-02T02:43:07.978009954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:49216"
ts=2026-05-02T02:43:10.516581149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T02:43:36.951114358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:41814"
ts=2026-05-02T02:43:40.517665961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T02:44:05.374820494Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-02T02:44:10.518881334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T02:44:36.95412081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:57892"
ts=2026-05-02T02:44:37.981754285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:47342"
ts=2026-05-02T02:44:40.520564977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T02:45:01.525791836Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-02T02:45:06.95573412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:51678"
ts=2026-05-02T02:45:10.521921989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T02:45:40.523610576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T02:46:01.131069859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-02T02:46:07.985944875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:33892"
ts=2026-05-02T02:46:10.524721393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T02:46:40.526114672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T02:47:06.9609411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:52470"
ts=2026-05-02T02:47:07.988225589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:35382"
level=debug ts=2026-05-02T02:47:10.084337205Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-02T02:47:10.085881009Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T02:47:10.527586052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T02:47:22.768712316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:38240"
ts=2026-05-02T02:47:37.989075953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:60314"
ts=2026-05-02T02:47:40.528550177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T02:48:06.964247825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:49792"
level=debug ts=2026-05-02T02:48:07.60201996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T02:48:10.530109616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T02:48:37.990828569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:34050"
ts=2026-05-02T02:48:40.531396274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T02:48:53.87313034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T02:49:07.992298485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:47702"
ts=2026-05-02T02:49:10.532692315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T02:49:22.774844251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:56444"
level=debug ts=2026-05-02T02:49:29.418709303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T02:49:40.534024875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T02:50:06.96961364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:55264"
ts=2026-05-02T02:50:10.535052152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T02:50:22.778038643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43716"
level=debug ts=2026-05-02T02:50:24.983151054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-02T02:50:32.054759892Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-02T02:50:32.057242409Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T02:50:32.059057616Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T02:50:32.060802144Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T02:50:32.060835818Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-02T02:50:32.060847803Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-02T02:50:36.971202378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:36144"
ts=2026-05-02T02:50:40.536367622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T02:50:52.779842227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:50642"
ts=2026-05-02T02:51:10.538215328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T02:51:13.07882339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T02:51:13.080400765Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T02:51:40.539348584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T02:51:52.782267975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:46676"
level=debug ts=2026-05-02T02:52:09.822810692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T02:52:10.541083822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T02:52:22.783271944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35578"
ts=2026-05-02T02:52:40.542606426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T02:52:49.252357718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T02:53:10.543772997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T02:53:36.979981995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:32790"
ts=2026-05-02T02:53:40.545151952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T02:53:40.94661766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T02:54:10.546264672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T02:54:22.829967069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T02:54:38.008581723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:50398"
ts=2026-05-02T02:54:40.548211725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T02:55:10.549525168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T02:55:24.724964957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T02:55:24.727538763Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T02:55:40.551431882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T02:56:06.986601136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:33720"
ts=2026-05-02T02:56:10.553051566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T02:56:21.088037302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T02:56:22.793401303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:40754"
ts=2026-05-02T02:56:36.988402496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:34878"
ts=2026-05-02T02:56:38.014624397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:46654"
ts=2026-05-02T02:56:40.554808413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T02:57:08.015908318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:34692"
ts=2026-05-02T02:57:10.556069128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T02:57:15.106408109Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T02:57:38.016950628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:54912"
ts=2026-05-02T02:57:40.557324799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T02:58:10.558427047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T02:58:15.26016062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T02:58:22.798769232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:44262"
ts=2026-05-02T02:58:40.560117019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T02:58:50.921353396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T02:58:52.799844802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54956"
ts=2026-05-02T02:59:06.995567423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:44072"
ts=2026-05-02T02:59:10.561860917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T02:59:40.563022515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T02:59:51.668125877Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T02:59:51.669722905Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:00:06.99871853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:54274"
ts=2026-05-02T03:00:08.023418228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:35864"
ts=2026-05-02T03:00:10.564676776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:00:22.803250638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48542"
level=error ts=2026-05-02T03:00:32.06569193Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:00:32.067831995Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:00:32.069896637Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:00:32.069922176Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:00:38.024833938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:47894"
ts=2026-05-02T03:00:40.566373152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:00:47.784183062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:01:07.002001951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:39992"
ts=2026-05-02T03:01:08.025978478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:42578"
ts=2026-05-02T03:01:10.567887684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:01:30.745480282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:01:40.569454399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:02:10.571138026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:02:22.808142853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:57038"
level=debug ts=2026-05-02T03:02:32.254617175Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:02:40.572679456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T03:02:52.809670436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:39406"
ts=2026-05-02T03:03:07.007791506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:57678"
ts=2026-05-02T03:03:10.573698607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:03:12.841852121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:03:40.575176308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:03:48.946179869Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T03:03:48.948009946Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:03:52.81291221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54854"
ts=2026-05-02T03:04:07.010916441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:52634"
ts=2026-05-02T03:04:10.57601827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:04:35.081385139Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:04:37.011893165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:53934"
ts=2026-05-02T03:04:40.577782543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T03:05:07.013605738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:51206"
level=debug ts=2026-05-02T03:05:10.322999756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:05:10.578872639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T03:05:38.037883115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:52438"
ts=2026-05-02T03:05:40.579729845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T03:06:10.58134006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T03:06:12.183304631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:06:37.01689931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:43472"
ts=2026-05-02T03:06:38.040944879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:48310"
ts=2026-05-02T03:06:40.582917265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:07:10.58439123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T03:07:11.342044383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:07:22.822393482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:59086"
ts=2026-05-02T03:07:37.019550324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:59438"
ts=2026-05-02T03:07:40.585332337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:08:00.466011384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T03:08:00.467589905Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:08:07.020912713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:39632"
ts=2026-05-02T03:08:10.586825163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T03:08:40.588427593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:08:52.060896417Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:08:52.826991807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47816"
ts=2026-05-02T03:09:10.589835534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:09:40.590921615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T03:09:44.451957991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:10:10.592312475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:10:22.832098863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:39420"
level=error ts=2026-05-02T03:10:32.064617667Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:10:32.066601697Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:10:32.068407939Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:10:32.068429126Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-02T03:10:39.015138481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:10:40.5938083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:11:10.594682029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T03:11:20.157539781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:11:22.835141159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:57174"
ts=2026-05-02T03:11:40.595693652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:11:59.637050166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T03:11:59.639106624Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:12:10.596946811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:12:37.036372387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:55926"
ts=2026-05-02T03:12:38.058979392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:33014"
ts=2026-05-02T03:12:40.598821884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:12:58.237934766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:13:10.600531517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T03:13:22.840335056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:46638"
ts=2026-05-02T03:13:37.040359016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:52764"
ts=2026-05-02T03:13:38.062399087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:50512"
ts=2026-05-02T03:13:40.60240648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:13:51.845570091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:14:10.6032755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T03:14:22.843492959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:51098"
level=debug ts=2026-05-02T03:14:26.932830408Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:14:38.065425275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:41410"
ts=2026-05-02T03:14:40.604256384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:15:07.044133979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:39618"
ts=2026-05-02T03:15:08.066994845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:47472"
ts=2026-05-02T03:15:10.605856131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:15:14.182000611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:15:37.045970938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:43462"
ts=2026-05-02T03:15:40.60726815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:16:07.892176254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T03:16:07.894236495Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:16:10.608886377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:16:22.849151688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:60262"
ts=2026-05-02T03:16:40.610148946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:17:09.92533492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:17:10.611445518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:17:40.612584884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T03:17:49.384562346Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:18:10.613869561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:18:38.078471934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:57274"
ts=2026-05-02T03:18:40.615279538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:18:51.672693029Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:18:52.85529568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:46742"
ts=2026-05-02T03:19:08.079581967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:60668"
ts=2026-05-02T03:19:10.616355962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T03:19:40.304446776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:19:40.61765685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T03:20:07.058051522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:50908"
ts=2026-05-02T03:20:08.082842903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:48112"
ts=2026-05-02T03:20:10.618748809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T03:20:22.859315712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:53714"
level=error ts=2026-05-02T03:20:32.06655619Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:20:32.0687242Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:20:32.070619583Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:20:32.070670997Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-02T03:20:37.009751526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T03:20:37.011542482Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:20:38.084379864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:33592"
ts=2026-05-02T03:20:40.620527723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T03:21:07.060313537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:51350"
ts=2026-05-02T03:21:10.621602674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T03:21:22.864923184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:51538"
level=debug ts=2026-05-02T03:21:26.566913786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:21:38.087011445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:53438"
ts=2026-05-02T03:21:40.623351835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:22:06.478888308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:22:08.088433044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:58814"
ts=2026-05-02T03:22:10.624951236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:22:40.625859648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T03:22:57.115663372Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:23:07.065266341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:56752"
ts=2026-05-02T03:23:10.627560144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T03:23:37.066923401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:36748"
ts=2026-05-02T03:23:40.62885651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:23:42.899098708Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:23:52.871482061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:40680"
ts=2026-05-02T03:24:10.630035271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:24:31.857345037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T03:24:31.859020848Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:24:38.094426324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:48592"
ts=2026-05-02T03:24:40.631025493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T03:24:52.873665592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:36694"
ts=2026-05-02T03:25:10.632303528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:25:19.455056335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:25:22.875127091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:59372"
ts=2026-05-02T03:25:37.072331024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:36430"
ts=2026-05-02T03:25:38.096596932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:33102"
ts=2026-05-02T03:25:40.633619329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T03:26:10.635337401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:26:18.725980139Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:26:40.636974739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:26:52.87875233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:36300"
level=debug ts=2026-05-02T03:27:01.072901102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:27:08.100450369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:32796"
ts=2026-05-02T03:27:10.638750769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:27:39.639071955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:27:40.639975808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:28:10.641295715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:28:16.628823316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T03:28:16.630501493Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:28:22.889464538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47288"
ts=2026-05-02T03:28:37.082608831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:53028"
ts=2026-05-02T03:28:40.643218973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:29:07.084396748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:58756"
ts=2026-05-02T03:29:10.64412573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:29:13.309970482Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:29:40.645433385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:29:49.887545565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:30:07.086686578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:33362"
ts=2026-05-02T03:30:08.108127801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:45660"
ts=2026-05-02T03:30:10.646269541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=error ts=2026-05-02T03:30:32.064242507Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:30:32.066277416Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:30:32.067751942Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:30:32.067770102Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:30:37.088082799Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:39544"
ts=2026-05-02T03:30:38.108976429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:46294"
level=debug ts=2026-05-02T03:30:39.30853313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:30:40.647980232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:31:08.110395592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:60636"
ts=2026-05-02T03:31:10.648939676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T03:31:37.090975058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:42308"
level=debug ts=2026-05-02T03:31:40.200391123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:31:40.650783678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T03:31:52.899063316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:40988"
ts=2026-05-02T03:32:08.112497796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:45010"
ts=2026-05-02T03:32:10.651950159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:32:28.511846676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T03:32:28.513279595Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:32:40.653062326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:33:04.97900215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:33:10.654860788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T03:33:37.09785619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:44056"
ts=2026-05-02T03:33:38.116440923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:36894"
ts=2026-05-02T03:33:40.65609004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:33:47.66016866Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:33:52.904324911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:58344"
ts=2026-05-02T03:34:07.099632262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:47760"
ts=2026-05-02T03:34:08.117785744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:60116"
ts=2026-05-02T03:34:10.657607879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:34:35.275953712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:34:40.658880829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T03:35:10.66007921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:35:29.236155121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:35:37.104460421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:38422"
ts=2026-05-02T03:35:40.661501988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:35:52.911058552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:39924"
ts=2026-05-02T03:36:10.662968976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:36:16.544430732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T03:36:16.546320502Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:36:37.107707947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:60222"
ts=2026-05-02T03:36:38.124909952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:57146"
ts=2026-05-02T03:36:40.664520827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:37:04.291171913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:37:07.109519995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:35468"
ts=2026-05-02T03:37:10.665492368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T03:37:22.915076364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:50812"
ts=2026-05-02T03:37:40.667322243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:37:58.509626272Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:38:07.112103206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:46384"
ts=2026-05-02T03:38:08.129668031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:48174"
ts=2026-05-02T03:38:10.668516943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:38:40.670116321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:38:59.850174309Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:39:07.114475113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:37292"
ts=2026-05-02T03:39:10.671722004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:39:40.019349189Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:39:40.673248968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:40:08.136051094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:57242"
ts=2026-05-02T03:40:10.674282929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=error ts=2026-05-02T03:40:32.065510502Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:40:32.06729676Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:40:32.068886426Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:40:32.068908387Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-02T03:40:36.980382316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T03:40:36.981915448Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:40:38.137232129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:51952"
ts=2026-05-02T03:40:40.675870224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T03:40:52.926324183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:59678"
ts=2026-05-02T03:41:07.119682408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:52662"
ts=2026-05-02T03:41:10.67687273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:41:26.5722318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:41:40.678663212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T03:41:52.929792133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:41126"
ts=2026-05-02T03:42:10.680493252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T03:42:26.437669294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:42:40.682273184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:42:52.932961321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:46992"
level=debug ts=2026-05-02T03:43:05.078610095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:43:08.144513384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:43392"
ts=2026-05-02T03:43:10.683902094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T03:43:22.933921316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35382"
ts=2026-05-02T03:43:40.684885263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:43:41.406472535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:44:10.685900883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:44:22.423462171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T03:44:22.425129929Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:44:22.937179767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:38142"
ts=2026-05-02T03:44:37.128828135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:52928"
ts=2026-05-02T03:44:40.687579721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T03:45:10.689143557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:45:20.514687359Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:45:22.939406274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:41726"
ts=2026-05-02T03:45:40.690185186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:45:52.940805091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:59206"
ts=2026-05-02T03:46:07.132785419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:42600"
ts=2026-05-02T03:46:10.691715645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:46:21.183820911Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:46:40.692635341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:47:07.136129755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:56256"
ts=2026-05-02T03:47:08.155918951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:51112"
level=debug ts=2026-05-02T03:47:08.877711371Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:47:10.694036963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T03:47:37.137894458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:48366"
ts=2026-05-02T03:47:40.694978535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:47:57.038925013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:48:07.138926821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:47192"
ts=2026-05-02T03:48:10.696074323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T03:48:39.787412192Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T03:48:39.788897796Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:48:40.69781343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T03:48:52.950134994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35902"
ts=2026-05-02T03:49:08.162094935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:58586"
ts=2026-05-02T03:49:10.698789704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T03:49:18.217972725Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:49:22.951924468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:50928"
ts=2026-05-02T03:49:40.700378707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:50:08.04141205Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:50:10.701907949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=error ts=2026-05-02T03:50:32.064553685Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:50:32.066734055Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:50:32.068530174Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T03:50:32.068555355Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:50:37.145799708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:55914"
ts=2026-05-02T03:50:40.703490705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:51:04.655521866Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:51:07.146634673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:60532"
ts=2026-05-02T03:51:08.167528803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:45402"
ts=2026-05-02T03:51:10.704530712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:51:39.750697381Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:51:40.705684847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T03:52:07.148810536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:52730"
ts=2026-05-02T03:52:10.707138678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T03:52:25.443084957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T03:52:25.44477565Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:52:40.708529589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T03:52:52.963063873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:46470"
ts=2026-05-02T03:53:10.709767133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:53:24.577252963Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:53:37.152236578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:40754"
ts=2026-05-02T03:53:40.711605617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
level=debug ts=2026-05-02T03:54:09.402724441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:54:10.712796664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
ts=2026-05-02T03:54:22.96691202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:42914"
ts=2026-05-02T03:54:37.154905114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:59664"
ts=2026-05-02T03:54:40.714496509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T03:54:52.968167254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:46430"
level=debug ts=2026-05-02T03:54:53.92891923Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:55:07.156511581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:47168"
ts=2026-05-02T03:55:08.177019141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:38696"
ts=2026-05-02T03:55:10.715974731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T03:55:22.96931468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:44278"
ts=2026-05-02T03:55:40.717712784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:55:53.164406734Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:56:08.178853506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:51522"
ts=2026-05-02T03:56:10.71881256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:56:29.974249328Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T03:56:29.975913481Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T03:56:40.720488001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:57:07.162865471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:48210"
ts=2026-05-02T03:57:08.18201068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:58162"
ts=2026-05-02T03:57:10.721832816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:57:24.278679832Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:57:40.723290748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:58:10.724298353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d268ff4 10.130.0.79:7946"
level=debug ts=2026-05-02T03:58:17.963905823Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:58:40.72535615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
ts=2026-05-02T03:59:10.7266278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wts6w-6a30553d 10.130.0.77:7946"
level=debug ts=2026-05-02T03:59:13.860004955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T03:59:22.980886884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:49478"
ts=2026-05-02T03:59:37.170317037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:49308"
ts=2026-05-02T03:59:38.189052094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:54990"
ts=2026-05-02T03:59:40.727507203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T03:59:52.981802826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47842"
level=debug ts=2026-05-02T04:00:03.564761953Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T04:00:10.729233351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xg2r6-adc3e096 10.130.0.75:7946"
ts=2026-05-02T04:00:22.982705626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:34738"
level=error ts=2026-05-02T04:00:32.064793538Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T04:00:32.066737196Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T04:00:32.068568072Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T04:00:32.06859373Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T04:00:37.173669543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:42950"