level=info ts=2026-05-07T14:55:09.962475117Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-07T14:55:09.96249548Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-07T14:55:09.96388376Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-07T14:55:09.964148175Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T14:55:09.964213819Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.83 inf=eth0
level=debug ts=2026-05-07T14:55:09.964617836Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T14:55:09.964673043Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.83 inf=eth0
level=info ts=2026-05-07T14:55:09.964773465Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-99347fe9
ts=2026-05-07T14:55:09.964825076Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-07T14:55:09.964834281Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.83 advertisePort=7946
level=debug ts=2026-05-07T14:55:09.964850226Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.83 advertisePort=7946
level=info ts=2026-05-07T14:55:09.964949305Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-07T14:55:09.971312082Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T14:55:09.971329222Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:55:09.971340578Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.392793ms
level=error ts=2026-05-07T14:55:09.971348661Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-07T14:55:09.971359581Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-07T14:55:09.974155284Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T14:55:09.974170598Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-07T14:55:09.979498285Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-05-07T14:55:09.979522003Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-07T14:55:09.979517732Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-07T14:55:09.979531069Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-07T14:55:09.979524278Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-07T14:55:09.979606291Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-05-07T14:55:09.979617848Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-07T14:55:09.979630313Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-07T14:55:09.97980916Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-07T14:55:09.979822762Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-07T14:55:09.9798559Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-07T14:55:09.979876666Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-07T14:55:09.979905623Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-07T14:55:09.980042836Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-07T14:55:10.170336991Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=info ts=2026-05-07T14:55:10.17045846Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-07T14:55:10.30668671Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-07T14:55:10.306740432Z caller=loki.go:615 msg="Loki started" startup_time=407.052345ms
ts=2026-05-07T14:55:11.211305495Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T14:55:11.211330975Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:55:15.062436594Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T14:55:15.062458378Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-07T14:55:15.307106538Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-07T14:55:15.307166049Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-07T14:55:21.882769631Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T14:55:21.882791256Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-07T14:55:24.210732522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:55:28.916450481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:55418"
ts=2026-05-07T14:55:34.062398486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:56842"
ts=2026-05-07T14:55:34.073843669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:59834"
ts=2026-05-07T14:55:36.316473173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.83:7946"
ts=2026-05-07T14:55:36.316588727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:36892"
ts=2026-05-07T14:55:36.317033828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.81:7946"
ts=2026-05-07T14:55:36.318541142Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.65:7946: dial tcp 10.128.0.65:7946: connect: connection refused"
ts=2026-05-07T14:55:36.319459686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.66:7946"
ts=2026-05-07T14:55:36.322161402Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.83:7946: dial tcp 10.130.0.83:7946: connect: connection refused"
level=info ts=2026-05-07T14:55:36.322172736Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=26.350807996s
level=debug ts=2026-05-07T14:55:44.726807225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:55:56.72365194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:44882"
ts=2026-05-07T14:55:59.014214906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T14:55:59.781634084Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:56:04.867358162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:46536"
level=debug ts=2026-05-07T14:56:20.578563537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:56:29.015521578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T14:56:51.298160612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T14:56:51.29977258Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T14:56:59.016565683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T14:57:20.106052542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:51482"
ts=2026-05-07T14:57:29.017791985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T14:57:34.876232488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:52226"
level=debug ts=2026-05-07T14:57:35.840918053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:57:59.018921237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T14:58:04.878939084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50440"
level=debug ts=2026-05-07T14:58:24.213231382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:58:29.021797289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T14:58:34.881716971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:44188"
ts=2026-05-07T14:58:59.024023646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T14:59:25.400732455Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:59:26.740430252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34380"
ts=2026-05-07T14:59:29.025426246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T14:59:34.888472203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47380"
ts=2026-05-07T14:59:59.026700701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:00:29.028489991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:00:34.893957469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:41380"
level=debug ts=2026-05-07T15:00:47.226529242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T15:00:50.12413958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:37788"
ts=2026-05-07T15:00:59.03181148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:01:04.897538429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:34098"
ts=2026-05-07T15:01:29.03341391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:01:34.900863908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:38744"
ts=2026-05-07T15:01:50.128994235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:37832"
ts=2026-05-07T15:01:56.749133037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:40200"
level=debug ts=2026-05-07T15:01:57.891063378Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T15:01:57.89294209Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:01:59.034418212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:02:04.903449475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:53296"
ts=2026-05-07T15:02:20.130704893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:56342"
ts=2026-05-07T15:02:29.036689455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:02:35.104106189Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:02:59.038771567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T15:03:13.81285185Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:03:26.752365469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:43922"
ts=2026-05-07T15:03:29.03996387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T15:03:54.077341446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:03:59.041209004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:04:29.042452978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:04:34.917323135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:35862"
level=debug ts=2026-05-07T15:04:36.354886541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:04:59.043456876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=info ts=2026-05-07T15:05:15.307751061Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-07T15:05:15.311371615Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:05:15.313133475Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:05:15.31486721Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:05:15.314892144Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T15:05:15.314903784Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-07T15:05:29.045022163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T15:05:31.319892713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:05:31.321951121Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:05:34.923877765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:54860"
ts=2026-05-07T15:05:59.046751447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:06:07.359580692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:06:26.763666646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:54216"
ts=2026-05-07T15:06:29.050056619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:06:55.300576035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:06:59.053364782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:07:26.766115244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:50782"
ts=2026-05-07T15:07:29.055973631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:07:34.938317839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:58072"
level=debug ts=2026-05-07T15:07:38.72394428Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:07:50.158240592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:54522"
ts=2026-05-07T15:07:59.059397902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:08:26.768956072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34522"
ts=2026-05-07T15:08:29.062523225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:08:37.580627327Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:08:59.064619592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:09:29.046545626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:09:29.048382179Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:09:29.065806279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:09:59.067872912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:10:04.953667399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:34358"
level=debug ts=2026-05-07T15:10:11.779371642Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:10:26.775990715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:51758"
ts=2026-05-07T15:10:29.069845114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:10:50.172102004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:40476"
level=debug ts=2026-05-07T15:10:58.698848356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:10:59.071018501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:11:20.172918474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:49802"
ts=2026-05-07T15:11:26.778528553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:56248"
ts=2026-05-07T15:11:29.072813963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:11:34.972292861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:49352"
level=debug ts=2026-05-07T15:11:54.301702463Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:11:59.074133552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:12:29.075697799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:12:36.506707012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:12:56.781496276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:56738"
ts=2026-05-07T15:12:59.077614947Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:13:11.621005157Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:13:11.623004334Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:13:20.180174522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:36292"
ts=2026-05-07T15:13:29.080987912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:13:34.992994292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:45680"
ts=2026-05-07T15:13:50.181621482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:39928"
ts=2026-05-07T15:13:59.082887917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:14:02.817071685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:14:04.996231924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:55596"
ts=2026-05-07T15:14:29.08547049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:14:59.088677706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:15:04.753660271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:15:05.002598525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:35750"
level=error ts=2026-05-07T15:15:15.319385283Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:15:15.321542005Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:15:15.323659791Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:15:15.32368114Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:15:20.189150223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:39232"
ts=2026-05-07T15:15:26.790659333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58924"
ts=2026-05-07T15:15:29.090865525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:15:50.190544716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:41246"
level=debug ts=2026-05-07T15:15:50.911496961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:15:59.093571049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:16:05.008553889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50158"
ts=2026-05-07T15:16:20.191454015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:46672"
ts=2026-05-07T15:16:26.794426411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58018"
ts=2026-05-07T15:16:29.095793728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:16:35.011736127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:45714"
level=debug ts=2026-05-07T15:16:40.412221089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:16:56.795678735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:41974"
ts=2026-05-07T15:16:59.097755368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:17:29.098769688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T15:17:34.030915603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:17:34.032783613Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:17:35.017869314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50552"
ts=2026-05-07T15:17:59.101520482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:18:19.560384256Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:18:20.200919077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:36072"
ts=2026-05-07T15:18:29.105041794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:18:50.20381622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:59792"
ts=2026-05-07T15:18:56.805194704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:37410"
ts=2026-05-07T15:18:59.106768282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:19:04.023682317Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:19:29.108455907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:19:35.031057662Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:45108"
level=debug ts=2026-05-07T15:19:41.28540492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:19:56.809023404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:49778"
ts=2026-05-07T15:19:59.109642797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:20:05.03459595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:41458"
ts=2026-05-07T15:20:26.810164405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:55948"
ts=2026-05-07T15:20:29.11093423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T15:20:35.114799651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:20:50.227424897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:52776"
ts=2026-05-07T15:20:59.112632042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:21:05.041045189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:35354"
ts=2026-05-07T15:21:29.1136735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:21:33.96418927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:21:33.966055568Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:21:35.050959098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:37100"
ts=2026-05-07T15:21:59.114661291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:22:10.266301991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:22:26.817470604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:38062"
ts=2026-05-07T15:22:29.116544423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:22:52.619222449Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:22:56.819126903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:33722"
ts=2026-05-07T15:22:59.118600464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:23:05.061102518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:33994"
ts=2026-05-07T15:23:26.820467856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:45024"
ts=2026-05-07T15:23:29.120349772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:23:35.063783747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:49074"
level=debug ts=2026-05-07T15:23:53.276422561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:23:59.122525883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:24:26.822835406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:41650"
ts=2026-05-07T15:24:29.124809631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:24:49.070472379Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:24:50.248891267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:37272"
ts=2026-05-07T15:24:56.824282708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:37470"
ts=2026-05-07T15:24:59.126058823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:25:05.073274638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:60632"
level=error ts=2026-05-07T15:25:15.321871517Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:25:15.323975482Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:25:15.32586911Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:25:15.325894017Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:25:20.250806907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:57692"
ts=2026-05-07T15:25:29.127762075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:25:46.390324693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:25:46.391837169Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:25:56.828406217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48706"
ts=2026-05-07T15:25:59.129615383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:26:05.078557632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:42524"
ts=2026-05-07T15:26:29.131664853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:26:32.991851787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:26:56.832995264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48126"
ts=2026-05-07T15:26:59.134566652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:27:20.254705882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:37186"
level=debug ts=2026-05-07T15:27:25.524947254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:27:26.834987813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:42514"
ts=2026-05-07T15:27:29.137511756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:27:35.088153848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:43514"
ts=2026-05-07T15:27:59.139556112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:28:20.258172584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:47412"
level=debug ts=2026-05-07T15:28:22.664011801Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:28:26.837561243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:44156"
ts=2026-05-07T15:28:29.141174984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:28:50.259663418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:40116"
ts=2026-05-07T15:28:59.142698985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:29:17.893857892Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:29:20.261454578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:37946"
ts=2026-05-07T15:29:29.144399177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:29:35.100999099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:55958"
ts=2026-05-07T15:29:59.145794603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:30:08.007549007Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:30:08.009024496Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:30:29.146924548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:30:35.107479224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:55112"
level=debug ts=2026-05-07T15:30:45.237046589Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:30:56.846557498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48592"
ts=2026-05-07T15:30:59.149209349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:31:20.269378407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:47470"
ts=2026-05-07T15:31:26.847504919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:35354"
ts=2026-05-07T15:31:29.152056973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:31:35.113112687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:56564"
level=debug ts=2026-05-07T15:31:43.654172225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:31:59.154190766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:32:29.155742803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:32:35.120066929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:45266"
level=debug ts=2026-05-07T15:32:41.732548234Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:32:59.157111701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:33:29.15863903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T15:33:40.720837187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:33:50.282158715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:37664"
ts=2026-05-07T15:33:56.856632951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:53858"
ts=2026-05-07T15:33:59.15977174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:34:29.161176844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T15:34:32.018915684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:34:32.02109743Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:34:50.284542434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:37898"
ts=2026-05-07T15:34:59.162741515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:35:05.135073849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:34508"
level=debug ts=2026-05-07T15:35:12.177436265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:35:15.317575427Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:35:15.319416087Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:35:15.321128685Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:35:15.321149556Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:35:29.165318936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:35:35.137781601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:52242"
level=debug ts=2026-05-07T15:35:55.056673905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:35:56.864412112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:55826"
ts=2026-05-07T15:35:59.168368766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:36:20.288338465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:34188"
ts=2026-05-07T15:36:26.865596415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:46890"
ts=2026-05-07T15:36:29.170766176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:36:50.290060561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:48594"
level=debug ts=2026-05-07T15:36:52.835783933Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:36:59.171696225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:37:05.146497605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:59392"
ts=2026-05-07T15:37:26.86988318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:43998"
ts=2026-05-07T15:37:29.173817206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:37:51.784087479Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:37:56.87117336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48948"
ts=2026-05-07T15:37:59.176894077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:38:26.872839302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:47452"
ts=2026-05-07T15:38:29.180356493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:38:46.959302593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:38:46.96092429Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:38:50.299112668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:40318"
ts=2026-05-07T15:38:59.182685222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:39:05.158778761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:56188"
ts=2026-05-07T15:39:20.300217632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:38856"
ts=2026-05-07T15:39:29.184001136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T15:39:48.160805263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:39:59.185386532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:40:29.187219638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:40:30.399562862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:40:50.304657792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:60400"
ts=2026-05-07T15:40:56.880588766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:50568"
ts=2026-05-07T15:40:59.189340171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:41:05.169318031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:33296"
level=debug ts=2026-05-07T15:41:08.226201559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:41:26.88161461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:38998"
ts=2026-05-07T15:41:29.190810607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:41:55.540635402Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:41:59.191865211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:42:20.310483517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:36518"
ts=2026-05-07T15:42:29.193017611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:42:35.177691165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:41030"
level=debug ts=2026-05-07T15:42:36.298305439Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:42:36.300385698Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:42:50.311619632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:34028"
ts=2026-05-07T15:42:56.888312351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:35348"
ts=2026-05-07T15:42:59.195232503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:43:29.197325885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T15:43:29.20913305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:43:59.19905195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:44:05.186459206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:33750"
level=debug ts=2026-05-07T15:44:19.298825545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:44:20.319039151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:58910"
ts=2026-05-07T15:44:26.892180289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:43298"
ts=2026-05-07T15:44:29.200926635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:44:50.320300357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:54974"
ts=2026-05-07T15:44:59.202755153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:45:03.044241291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:45:05.192738499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:57092"
level=error ts=2026-05-07T15:45:15.318806786Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:45:15.320538333Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:45:15.322146124Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:45:15.3221697Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:45:20.32135758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:46504"
ts=2026-05-07T15:45:29.205439473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:45:44.396230901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:45:56.897940331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:47024"
ts=2026-05-07T15:45:59.207680823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:46:20.32529677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:50818"
level=debug ts=2026-05-07T15:46:24.293044065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:46:24.295038246Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:46:29.208890523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:46:59.210459493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:47:01.790193935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:47:26.904544976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:56648"
ts=2026-05-07T15:47:29.211713997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:47:35.207404922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:48784"
level=debug ts=2026-05-07T15:47:46.395205294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:47:59.213415137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:48:26.907998629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:50004"
ts=2026-05-07T15:48:29.215221326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:48:35.213195699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:34562"
level=debug ts=2026-05-07T15:48:46.655587372Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:48:59.217834916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:49:20.3370028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:49562"
ts=2026-05-07T15:49:26.912718951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:35174"
ts=2026-05-07T15:49:29.220214222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:49:38.565032449Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:49:59.221747605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:50:05.22241657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:38802"
ts=2026-05-07T15:50:29.222805837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T15:50:37.125447467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:50:37.127048977Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:50:50.343132246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:53324"
ts=2026-05-07T15:50:59.225163353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:51:19.92451026Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:51:26.917658698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:40662"
ts=2026-05-07T15:51:29.227642094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T15:51:58.159837194Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:51:59.229108121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:52:26.920772041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:37036"
ts=2026-05-07T15:52:29.231782447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:52:39.873195695Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:52:50.350621408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:54644"
ts=2026-05-07T15:52:56.922675669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:33214"
ts=2026-05-07T15:52:59.234121098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:53:20.352486141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:49920"
level=debug ts=2026-05-07T15:53:26.721672156Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:53:29.235340667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:53:50.353926947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:35470"
ts=2026-05-07T15:53:59.237110364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:54:18.879650204Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:54:18.881504146Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:54:29.238214776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:54:55.920576641Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:54:59.239220824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=error ts=2026-05-07T15:55:15.319841913Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:55:15.32212337Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:55:15.324347844Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:55:15.324371239Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:55:20.361561786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:35250"
ts=2026-05-07T15:55:29.240357456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:55:34.486652019Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:55:56.936514652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:43282"
ts=2026-05-07T15:55:59.242807439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:56:16.546369245Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:56:29.244379181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:56:59.245337595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T15:57:01.306953553Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:57:29.247906692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T15:57:50.371182416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:35896"
level=debug ts=2026-05-07T15:57:58.693849471Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:57:58.695434626Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:57:59.250143239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T15:58:26.943869066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:45928"
ts=2026-05-07T15:58:29.252773015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:58:46.227715651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:58:50.375163785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:57460"
ts=2026-05-07T15:58:59.254366748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T15:59:29.255553042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:59:45.59051703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:59:59.25727227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:00:21.468968321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:00:29.259735307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T16:00:59.262098763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T16:01:22.786771841Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:01:26.958282811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:39126"
ts=2026-05-07T16:01:29.263494035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T16:01:50.386918275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:41058"
ts=2026-05-07T16:01:59.266081524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:02:01.495765918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:02:01.498035808Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:02:05.292698773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50786"
ts=2026-05-07T16:02:29.269343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T16:02:35.295326297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:37202"
ts=2026-05-07T16:02:50.391356027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:33846"
level=debug ts=2026-05-07T16:02:54.964235968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:02:59.271902823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T16:03:29.274127031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T16:03:33.774231935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:03:56.969679806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48112"
ts=2026-05-07T16:03:59.276092059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T16:04:29.277617934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T16:04:30.186197308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:04:50.399426932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:47930"
ts=2026-05-07T16:04:56.972387777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:55274"
ts=2026-05-07T16:04:59.278676149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T16:05:05.310248649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:46894"
level=debug ts=2026-05-07T16:05:08.315371766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:05:15.317902466Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:05:15.31957061Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:05:15.321199802Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:05:15.321224073Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:05:29.28012107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T16:05:49.79520879Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:05:49.796776831Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:05:50.402429481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:59632"
ts=2026-05-07T16:05:59.281580486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T16:06:29.282997601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T16:06:30.940092351Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:06:56.982581104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:37542"
ts=2026-05-07T16:06:59.284891922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T16:07:05.323194514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50198"
level=debug ts=2026-05-07T16:07:06.269427334Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:07:29.286577607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T16:07:35.326458032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:52302"
ts=2026-05-07T16:07:50.410110792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:53078"
level=debug ts=2026-05-07T16:07:55.254669066Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:07:56.986467647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:52156"
ts=2026-05-07T16:07:59.288991702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T16:08:20.411048442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:39046"
ts=2026-05-07T16:08:29.290539063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T16:08:35.332057901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:51124"
level=debug ts=2026-05-07T16:08:46.91543398Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:08:59.292466198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T16:09:05.334807442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:49800"
ts=2026-05-07T16:09:26.992479114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:36200"
ts=2026-05-07T16:09:29.2954987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:09:47.315014779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:09:47.316470931Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:09:59.297477229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T16:10:20.420472563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:37976"
ts=2026-05-07T16:10:26.995401403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58340"
ts=2026-05-07T16:10:29.298419779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T16:10:35.343480872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:42652"
level=debug ts=2026-05-07T16:10:38.352125015Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:10:50.42199044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:41454"
ts=2026-05-07T16:10:59.300085587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T16:11:29.30237373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:11:38.181865872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:11:50.424304394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:43988"
ts=2026-05-07T16:11:59.304179788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T16:12:05.352195459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:35534"
level=debug ts=2026-05-07T16:12:19.695854192Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:12:29.306488284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T16:12:50.42716042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:56746"
ts=2026-05-07T16:12:59.308528078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:13:00.853987088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:13:20.428303116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:42168"
ts=2026-05-07T16:13:27.007128918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:47016"
ts=2026-05-07T16:13:29.310561651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:13:45.744069327Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:13:45.745828935Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:13:57.007973252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:57984"
ts=2026-05-07T16:13:59.312681013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T16:14:05.364557142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:56592"
level=debug ts=2026-05-07T16:14:21.049920787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:14:27.008919575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:55100"
ts=2026-05-07T16:14:29.315086766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T16:14:59.31804386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:15:07.139460318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:15:15.318219618Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:15:15.319823298Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:15:15.321399025Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:15:15.321421211Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:15:27.01456541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:44236"
ts=2026-05-07T16:15:29.320841521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T16:15:35.373564088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:34294"
level=debug ts=2026-05-07T16:15:52.608645683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:15:59.322137295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T16:16:05.376710859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:35564"
ts=2026-05-07T16:16:20.437917248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:35084"
ts=2026-05-07T16:16:27.017841407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:33936"
ts=2026-05-07T16:16:29.323722574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T16:16:38.258622639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:16:50.439180104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:54602"
ts=2026-05-07T16:16:59.325439747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:17:15.323415785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:17:15.325023492Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:17:27.021211904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:59976"
ts=2026-05-07T16:17:29.326751475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T16:17:35.385511435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:45880"
level=debug ts=2026-05-07T16:17:54.897659444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:17:59.328409782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T16:18:29.33012611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:18:39.86134844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:18:57.027389473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:54528"
ts=2026-05-07T16:18:59.330985943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T16:19:23.593287177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:19:29.332860264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T16:19:35.396641868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50504"
ts=2026-05-07T16:19:50.453728328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:60118"
ts=2026-05-07T16:19:59.335504334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:20:02.175525876Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:20:29.337728229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T16:20:44.136910688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:20:44.138379109Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:20:50.458132705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:59998"
ts=2026-05-07T16:20:59.33865131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T16:21:20.459566743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:53300"
level=debug ts=2026-05-07T16:21:26.890539324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:21:27.035782812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48810"
ts=2026-05-07T16:21:29.339660638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T16:21:57.037383434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:56328"
ts=2026-05-07T16:21:59.341101559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T16:22:05.412293124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:42388"
level=debug ts=2026-05-07T16:22:09.844595013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:22:27.03920341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:56212"
ts=2026-05-07T16:22:29.34227601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T16:22:35.4149825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:37768"
level=debug ts=2026-05-07T16:22:51.537788406Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:22:59.343720523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T16:23:05.417865556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:42002"
ts=2026-05-07T16:23:27.0434327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:35102"
ts=2026-05-07T16:23:29.345385757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T16:23:30.962362981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:23:35.420499355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:37290"
ts=2026-05-07T16:23:59.347094142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:24:23.199342997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:24:23.200868733Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:24:27.047302192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:43122"
ts=2026-05-07T16:24:29.349221266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T16:24:57.048711651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:35694"
ts=2026-05-07T16:24:59.35114655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=error ts=2026-05-07T16:25:15.318789612Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:25:15.320719721Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:25:15.322445649Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:25:15.322470127Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:25:20.478353808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:57672"
level=debug ts=2026-05-07T16:25:24.35629982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:25:29.352706861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
ts=2026-05-07T16:25:57.051767902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:54874"
ts=2026-05-07T16:25:59.353719982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T16:26:05.434007701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:56222"
level=debug ts=2026-05-07T16:26:13.128466283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:26:29.354764195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:26:49.407165983Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:26:50.483456755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:40662"
ts=2026-05-07T16:26:59.356993808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T16:27:20.485320408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:40076"
level=debug ts=2026-05-07T16:27:29.196032773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:27:29.358593985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T16:27:35.442504592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:40502"
ts=2026-05-07T16:27:50.486767219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:57736"
ts=2026-05-07T16:27:59.359732155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T16:28:05.445675514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:39158"
ts=2026-05-07T16:28:20.488070306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:57142"
level=debug ts=2026-05-07T16:28:25.841068529Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:28:25.84293268Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:28:29.360688674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T16:28:59.363187645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:29:16.604963724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:29:20.493349389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:35318"
ts=2026-05-07T16:29:27.063380322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:42200"
ts=2026-05-07T16:29:29.365602882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
ts=2026-05-07T16:29:59.366971933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-66b5a331 10.129.0.82:7946"
level=debug ts=2026-05-07T16:30:04.40767702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:30:27.065994415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:33594"
ts=2026-05-07T16:30:29.36922711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:30:43.459601203Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:30:59.371802256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:31:26.397412486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:31:29.374887938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T16:31:59.378247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
ts=2026-05-07T16:32:05.469009853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:59050"