level=info ts=2026-05-06T06:50:24.796812757Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-06T06:50:24.796867964Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-06T06:50:24.799861634Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-06T06:50:24.800185281Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-06T06:50:24.800265876Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.88 inf=eth0
level=debug ts=2026-05-06T06:50:24.800667829Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-06T06:50:24.800741178Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.88 inf=eth0
level=info ts=2026-05-06T06:50:24.802061941Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-9d62f4be
ts=2026-05-06T06:50:24.802175245Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-06T06:50:24.802190841Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.88 advertisePort=7946
level=debug ts=2026-05-06T06:50:24.802212519Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.88 advertisePort=7946
level=info ts=2026-05-06T06:50:24.80239417Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-06T06:50:24.819919502Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-06T06:50:24.81994122Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-06T06:50:24.819965667Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=17.574556ms
level=error ts=2026-05-06T06:50:24.81997791Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-06T06:50:24.819990392Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-05-06T06:50:24.82126719Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-05-06T06:50:24.821308444Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-06T06:50:24.821379172Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-06T06:50:24.821475232Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-06T06:50:24.821547388Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-05-06T06:50:24.821563545Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-06T06:50:24.821833157Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-05-06T06:50:24.821841241Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-05-06T06:50:24.822072666Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-05-06T06:50:24.822311936Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-06T06:50:24.822387729Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-06T06:50:24.82243149Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-06T06:50:24.822457438Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-06T06:50:24.822640192Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-06T06:50:24.822651946Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-06T06:50:24.822666936Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-05-06T06:50:24.829151815Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T06:50:24.829171657Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-06T06:50:25.823625407Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-06T06:50:25.993749312Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-06T06:50:25.993793638Z caller=loki.go:615 msg="Loki started" startup_time=1.267909452s
ts=2026-05-06T06:50:26.733835786Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T06:50:26.733863621Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:50:28.96855037Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T06:50:28.968577014Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-06T06:50:30.993998435Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-06T06:50:30.994070882Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-06T06:50:34.624114651Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T06:50:34.624136356Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-06T06:50:44.326051636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:50:44.400247228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:50890"
ts=2026-05-06T06:50:45.444541273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56800"
ts=2026-05-06T06:50:46.626258377Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.87:7946: dial tcp 10.128.0.87:7946: connect: connection refused"
ts=2026-05-06T06:50:46.627036587Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.86:7946: dial tcp 10.128.0.86:7946: connect: connection refused"
ts=2026-05-06T06:50:46.627211058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.83:7946"
ts=2026-05-06T06:50:46.628391741Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.82:7946: dial tcp 10.128.0.82:7946: connect: connection refused"
ts=2026-05-06T06:50:46.628486041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.88:7946"
ts=2026-05-06T06:50:46.62852959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:35836"
ts=2026-05-06T06:50:46.629076973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.84:7946"
level=info ts=2026-05-06T06:50:46.629449982Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=21.809452328s
level=debug ts=2026-05-06T06:51:06.963245911Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:51:07.984541253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:49328"
ts=2026-05-06T06:51:19.390655787Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
level=debug ts=2026-05-06T06:51:30.520154545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:51:49.391487947Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
level=debug ts=2026-05-06T06:51:53.706433665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-06T06:52:17.874334969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-06T06:52:17.876028513Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T06:52:19.393071543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T06:52:21.756700643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:35624"
ts=2026-05-06T06:52:25.031913352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:52390"
ts=2026-05-06T06:52:49.394756947Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
level=debug ts=2026-05-06T06:52:53.96768552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:52:55.033265964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:43270"
ts=2026-05-06T06:53:19.396557865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T06:53:25.034653103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:52566"
ts=2026-05-06T06:53:49.398318288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T06:53:51.761389246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:48578"
level=debug ts=2026-05-06T06:54:12.820431406Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:54:19.399740882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T06:54:21.762757248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:49732"
ts=2026-05-06T06:54:49.401154332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T06:54:55.037933817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:52204"
ts=2026-05-06T06:55:19.402139816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T06:55:20.689489823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:42634"
ts=2026-05-06T06:55:21.765420371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:55560"
level=debug ts=2026-05-06T06:55:24.959577167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:55:25.039057701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:45504"
ts=2026-05-06T06:55:49.403066173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T06:55:55.040090442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:60948"
ts=2026-05-06T06:56:19.404438604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T06:56:20.692257758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:47118"
level=debug ts=2026-05-06T06:56:37.582595504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:56:49.405271797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T06:56:50.693260343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:41518"
ts=2026-05-06T06:57:19.406827334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T06:57:21.770929996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:52916"
ts=2026-05-06T06:57:25.043733015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:51176"
level=debug ts=2026-05-06T06:57:48.312762182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-06T06:57:48.31450876Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T06:57:49.407733906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T06:57:51.77266076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:39222"
ts=2026-05-06T06:57:55.04505398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:43348"
ts=2026-05-06T06:58:19.408808077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T06:58:21.774176051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:51594"
level=debug ts=2026-05-06T06:58:48.524658356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:58:49.410351566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T06:58:50.698575192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:44082"
ts=2026-05-06T06:58:55.048049558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:48462"
ts=2026-05-06T06:59:19.41179261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T06:59:21.776124001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:52816"
level=debug ts=2026-05-06T06:59:25.596405436Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:59:49.412707819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
level=debug ts=2026-05-06T07:00:11.342978072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:00:19.414238449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
level=info ts=2026-05-06T07:00:30.994835525Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-06T07:00:30.997925411Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:00:30.999668446Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:00:31.00125223Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:00:31.001276729Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-06T07:00:31.001288586Z caller=tables_manager.go:148 msg="compactor started"
level=debug ts=2026-05-06T07:00:45.951691996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:00:49.415759527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:00:51.780827454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:41890"
ts=2026-05-06T07:00:55.052422984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:54738"
ts=2026-05-06T07:01:19.417231948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:01:21.781805898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:40280"
ts=2026-05-06T07:01:25.053480737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:39946"
level=debug ts=2026-05-06T07:01:43.474689079Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:01:43.476302343Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:01:49.418593911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:01:50.706159689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:43786"
ts=2026-05-06T07:01:51.782701509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56688"
ts=2026-05-06T07:01:55.055060288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:50796"
ts=2026-05-06T07:02:19.420204942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T07:02:28.76073901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:02:49.421883694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:02:55.057631509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:47318"
level=debug ts=2026-05-06T07:03:17.14443312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:03:19.422888138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:03:49.424333766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T07:04:17.301402997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:04:19.42532444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:04:20.713087426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:49774"
ts=2026-05-06T07:04:49.426281193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:04:50.714624339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37426"
ts=2026-05-06T07:04:55.063917423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:46756"
level=debug ts=2026-05-06T07:05:06.795499259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:05:19.427277845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:05:21.791865621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:52880"
ts=2026-05-06T07:05:49.428830192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:05:50.718119867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:40734"
ts=2026-05-06T07:05:55.066276706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:39540"
level=debug ts=2026-05-06T07:06:07.549540771Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:06:07.551340907Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:06:19.430542808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
level=debug ts=2026-05-06T07:06:46.971155572Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:06:49.432141349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:07:19.43377778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:07:25.071095279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:38284"
level=debug ts=2026-05-06T07:07:38.238863711Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:07:49.435337157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:07:50.723437286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:45114"
ts=2026-05-06T07:08:19.436337839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
level=debug ts=2026-05-06T07:08:24.89718924Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:08:25.079621209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:35858"
ts=2026-05-06T07:08:49.437889274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T07:09:10.256346841Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:09:19.439431735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:09:49.440362156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T07:09:55.794260759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:09:55.796146676Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:10:19.441334162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:10:21.805885834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53770"
ts=2026-05-06T07:10:25.08650876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:33282"
level=error ts=2026-05-06T07:10:31.005817751Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:10:31.007838175Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:10:31.009728414Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:10:31.009752646Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-06T07:10:48.722036698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:10:49.443180578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:11:19.4442832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
level=debug ts=2026-05-06T07:11:39.513085127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:11:49.445354231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:12:19.446566163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:12:20.736728542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:43740"
ts=2026-05-06T07:12:21.812209601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53414"
level=debug ts=2026-05-06T07:12:31.451987542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:12:49.449839655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:12:55.09341546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:53648"
ts=2026-05-06T07:13:19.451395087Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:13:21.815305023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:40232"
level=debug ts=2026-05-06T07:13:23.490318597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:13:49.452267179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:13:55.096150749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:56712"
level=debug ts=2026-05-06T07:13:58.248182261Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:13:58.249846209Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:14:19.453885719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T07:14:38.878096962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:14:49.455372992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:14:50.744560854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:41326"
ts=2026-05-06T07:15:19.456474227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
level=debug ts=2026-05-06T07:15:23.365816502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:15:49.458156232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
level=debug ts=2026-05-06T07:16:13.97874738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:16:19.459770112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:16:20.749384682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:54042"
ts=2026-05-06T07:16:49.460921163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:16:55.104924381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:50806"
level=debug ts=2026-05-06T07:17:09.626808367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:17:19.46224298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:17:25.106241706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:58026"
ts=2026-05-06T07:17:49.463378602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:17:51.827504833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:46138"
level=debug ts=2026-05-06T07:18:11.538721192Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:18:11.540558757Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:18:19.464853648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:18:25.108964897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:42534"
ts=2026-05-06T07:18:49.466335714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
level=debug ts=2026-05-06T07:19:11.319362436Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:19:19.467364543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:19:49.468265871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:19:50.759692135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:52044"
level=debug ts=2026-05-06T07:19:54.285493225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:20:19.469314738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:20:20.76157359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:45040"
level=error ts=2026-05-06T07:20:31.005233701Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:20:31.007147478Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:20:31.008938591Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:20:31.008959539Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:20:49.470297398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:20:50.762779153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:58436"
level=debug ts=2026-05-06T07:20:54.655736999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:20:55.117891178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:53282"
ts=2026-05-06T07:21:19.471541927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:21:20.764549524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:54254"
level=debug ts=2026-05-06T07:21:48.353052588Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:21:49.473278206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:21:50.765590351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:60166"
ts=2026-05-06T07:21:51.839389986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:41854"
ts=2026-05-06T07:21:55.120825033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:35430"
ts=2026-05-06T07:22:19.474767632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:22:21.841253207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:43490"
ts=2026-05-06T07:22:25.123760083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:45916"
level=debug ts=2026-05-06T07:22:34.2988241Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:22:34.30041687Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:22:49.475870819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:22:50.768711338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:55816"
level=debug ts=2026-05-06T07:23:14.008528142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:23:19.476738247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:23:21.843868986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:33518"
level=debug ts=2026-05-06T07:23:47.951353972Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:23:49.477753171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:23:50.771119288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:56838"
ts=2026-05-06T07:23:55.129141938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:48688"
ts=2026-05-06T07:24:19.479299766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:24:25.130072773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:56138"
level=debug ts=2026-05-06T07:24:46.022207025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:24:49.48065855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:25:19.481424074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:25:20.775650627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:60452"
level=debug ts=2026-05-06T07:25:36.220575829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:25:49.482546074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:25:55.134247779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:33908"
ts=2026-05-06T07:26:19.483770617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:26:21.85182697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:36336"
level=debug ts=2026-05-06T07:26:28.939827303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:26:28.941797135Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:26:49.485873847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
level=debug ts=2026-05-06T07:27:13.440760347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:27:19.487520438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:27:49.48886222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T07:28:12.64768073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:28:19.490061935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
level=debug ts=2026-05-06T07:28:48.958619744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:28:49.491410104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:28:51.859772323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:42908"
ts=2026-05-06T07:29:19.492877139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:29:21.860881861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:33330"
level=debug ts=2026-05-06T07:29:32.177160469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:29:49.49437681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:29:51.862348742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:51244"
ts=2026-05-06T07:30:19.495977495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:30:20.791002191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:46812"
ts=2026-05-06T07:30:21.863747695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:46664"
level=debug ts=2026-05-06T07:30:30.421002464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:30:30.422711659Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:30:31.003786396Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:30:31.005793807Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:30:31.008107963Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:30:31.00813502Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:30:49.497591669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T07:31:05.823672662Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:31:19.499093339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:31:21.866228099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:39528"
level=debug ts=2026-05-06T07:31:47.54051712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:31:49.50056257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:31:50.795364784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:58190"
ts=2026-05-06T07:32:19.50181001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:32:21.870047703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:50780"
level=debug ts=2026-05-06T07:32:23.287942346Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:32:49.503589999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:32:50.798148941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:35482"
ts=2026-05-06T07:32:51.871345557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:44948"
level=debug ts=2026-05-06T07:33:00.825140085Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:33:19.504477446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T07:33:41.21791025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:33:41.219794032Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:33:49.505651891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:33:50.800641095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:45884"
ts=2026-05-06T07:33:55.15784775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:54442"
ts=2026-05-06T07:34:19.506766653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:34:25.159020763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:55014"
level=debug ts=2026-05-06T07:34:31.412485177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:34:49.507846088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:34:55.160694028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:52940"
ts=2026-05-06T07:35:19.510494868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T07:35:20.223526665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:35:25.162166164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:34242"
ts=2026-05-06T07:35:49.51224287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:35:50.805398957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:48170"
level=debug ts=2026-05-06T07:35:57.249988834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:36:19.513711304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
level=debug ts=2026-05-06T07:36:43.523560954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:36:49.51525665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:36:50.807924804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:40862"
ts=2026-05-06T07:37:19.517092951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:37:20.808945699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:51630"
ts=2026-05-06T07:37:21.882722157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:60816"
ts=2026-05-06T07:37:25.168267267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:56606"
level=debug ts=2026-05-06T07:37:32.68492172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:37:32.686743435Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:37:49.518340645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:37:51.884084043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56314"
ts=2026-05-06T07:38:19.520064849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
level=debug ts=2026-05-06T07:38:21.727213699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:38:49.522236425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T07:38:56.148150825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:39:19.524112325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:39:21.888711255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:49048"
level=debug ts=2026-05-06T07:39:44.713346568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:39:49.525669771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:39:51.890725762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:44870"
ts=2026-05-06T07:40:19.527115885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:40:20.817433526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:33012"
level=debug ts=2026-05-06T07:40:22.212517764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:40:25.176615301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:41928"
level=error ts=2026-05-06T07:40:31.003840443Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:40:31.005532775Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:40:31.007167703Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:40:31.007195255Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:40:49.528692031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:40:51.893283516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:54214"
ts=2026-05-06T07:40:55.177884491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:46468"
level=debug ts=2026-05-06T07:40:56.709970676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:40:56.711533909Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:41:19.530371824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:41:20.820464061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:33256"
ts=2026-05-06T07:41:21.895000285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:37622"
ts=2026-05-06T07:41:25.179462115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:38818"
level=debug ts=2026-05-06T07:41:40.047987141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:41:49.532162749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:42:19.533468336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T07:42:39.20226233Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:42:49.534544091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:42:50.824724973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38986"
ts=2026-05-06T07:43:19.536319187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T07:43:25.240912206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:43:49.53812537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:43:55.18567369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:56430"
ts=2026-05-06T07:44:19.540064285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
level=debug ts=2026-05-06T07:44:20.964538689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:44:49.541985406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T07:45:01.748946839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:45:01.751059474Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:45:19.543570542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T07:45:46.191323486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:45:49.545627328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:45:51.907516542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:49106"
ts=2026-05-06T07:45:55.192389291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:43524"
ts=2026-05-06T07:46:19.547018824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:46:21.909182583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56522"
ts=2026-05-06T07:46:25.193452976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:52756"
level=debug ts=2026-05-06T07:46:46.291743752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:46:49.54824457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:47:19.550358325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
level=debug ts=2026-05-06T07:47:37.996975807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:47:49.552277633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:47:50.840124038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:41546"
ts=2026-05-06T07:47:55.197977269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:59944"
ts=2026-05-06T07:48:19.554116934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
level=debug ts=2026-05-06T07:48:29.074671927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:48:49.555877588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:48:51.916020641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:44690"
ts=2026-05-06T07:49:19.556987345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:49:20.844344288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:57748"
level=debug ts=2026-05-06T07:49:27.618450654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:49:27.620020442Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:49:49.557867934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T07:50:12.935012157Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:50:19.558995943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:50:20.847821501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:53080"
ts=2026-05-06T07:50:25.20412687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:55586"
level=error ts=2026-05-06T07:50:31.00600402Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:50:31.007592812Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:50:31.009179378Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:50:31.009201684Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-06T07:50:47.783026552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:50:49.560008817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:50:50.848819251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:60536"
ts=2026-05-06T07:51:19.561581438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:51:21.922771564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53462"
level=debug ts=2026-05-06T07:51:36.897660885Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:51:49.563071653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:52:19.564943054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
level=debug ts=2026-05-06T07:52:28.964803234Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:52:49.566495543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:52:55.209485442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:43836"
level=debug ts=2026-05-06T07:53:06.259779834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:53:06.261364488Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:53:19.568028603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
level=debug ts=2026-05-06T07:53:44.866718997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:53:49.569040527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:53:51.930173438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:44622"
ts=2026-05-06T07:53:55.211869613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:56814"
ts=2026-05-06T07:54:19.56998472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
level=debug ts=2026-05-06T07:54:35.957692984Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:54:49.57149917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:54:51.932127507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:35746"
ts=2026-05-06T07:55:19.572972557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
level=debug ts=2026-05-06T07:55:23.607943612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:55:25.215008136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:46274"
ts=2026-05-06T07:55:49.574618288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:55:55.216529793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:40788"
ts=2026-05-06T07:56:19.576039505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
level=debug ts=2026-05-06T07:56:25.882379359Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:56:49.57755304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:56:55.219520218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:52676"
level=debug ts=2026-05-06T07:57:10.136030448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:57:10.137621318Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:57:19.578981609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:57:25.221059935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:41374"
level=debug ts=2026-05-06T07:57:46.19401194Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:57:49.580037136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:57:51.941200137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:50926"
ts=2026-05-06T07:57:55.223035474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:51956"
ts=2026-05-06T07:58:19.580973745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:58:21.942284462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:44784"
level=debug ts=2026-05-06T07:58:42.927493596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:58:49.58246224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T07:59:19.583595623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T07:59:20.872336438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:34446"
ts=2026-05-06T07:59:21.945146054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:32808"
ts=2026-05-06T07:59:25.226159593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:47326"
level=debug ts=2026-05-06T07:59:34.851545502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:59:49.585037023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T07:59:50.873438898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:49376"
ts=2026-05-06T07:59:51.946209102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:58700"
level=debug ts=2026-05-06T08:00:16.348694658Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:00:19.586572112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T08:00:25.227710912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:43868"
level=error ts=2026-05-06T08:00:31.005413273Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:00:31.007575738Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:00:31.00954448Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:00:31.009566781Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:00:49.587730651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
level=debug ts=2026-05-06T08:00:51.038800065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:00:51.040396468Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:01:19.589547675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T08:01:32.700430077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:01:49.590986793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T08:01:50.878113985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37014"
ts=2026-05-06T08:01:51.951248705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:54686"
ts=2026-05-06T08:02:19.592030634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T08:02:32.934210705Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:02:49.593121793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:02:51.954841371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:54734"
ts=2026-05-06T08:03:19.595099248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:03:21.956403733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:50284"
level=debug ts=2026-05-06T08:03:31.328428042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:03:49.596481204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T08:03:50.882835192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:40446"
ts=2026-05-06T08:04:19.597885121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:04:20.883952053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36764"
level=debug ts=2026-05-06T08:04:21.384813847Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:04:49.599651539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T08:04:51.95996138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53652"
ts=2026-05-06T08:04:55.23860324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:49854"
level=debug ts=2026-05-06T08:05:13.702384969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:05:13.703813392Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:05:19.601288632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T08:05:20.886629369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:59274"
ts=2026-05-06T08:05:49.602121306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
level=debug ts=2026-05-06T08:05:56.545510544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:06:19.603679522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T08:06:20.889564426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36202"
ts=2026-05-06T08:06:21.96402794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:55544"
ts=2026-05-06T08:06:25.24209864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:35786"
ts=2026-05-06T08:06:49.604941328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
level=debug ts=2026-05-06T08:06:52.277979484Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:07:19.606258518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T08:07:25.24403481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:44210"
level=debug ts=2026-05-06T08:07:48.065593059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:07:49.607750446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:07:50.892668695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:42502"
ts=2026-05-06T08:07:55.245604695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:43308"
ts=2026-05-06T08:08:19.608737076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T08:08:21.9694348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38844"
level=debug ts=2026-05-06T08:08:25.889932786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:08:49.609890089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T08:08:51.9706076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:44560"
ts=2026-05-06T08:08:55.248545495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:40850"
level=debug ts=2026-05-06T08:09:08.561601074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:09:08.563147093Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:09:19.61080432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T08:09:25.24985226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:39934"
level=debug ts=2026-05-06T08:09:48.107885112Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:09:49.611909204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:09:50.898677076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:40718"
ts=2026-05-06T08:09:51.974013466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:50612"
ts=2026-05-06T08:09:55.251333716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:56794"
ts=2026-05-06T08:10:19.612908665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T08:10:20.900046694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:55054"
ts=2026-05-06T08:10:21.975870776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:40164"
level=error ts=2026-05-06T08:10:31.005495152Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:10:31.007142139Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:10:31.008748064Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:10:31.008771329Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-06T08:10:35.820691039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:10:49.613913433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:11:19.615117137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:11:25.255476951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:54594"
level=debug ts=2026-05-06T08:11:29.975622124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:11:49.616927831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T08:11:51.979364576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38126"
level=debug ts=2026-05-06T08:12:12.828074227Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:12:19.617966165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T08:12:20.906691474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:32868"
ts=2026-05-06T08:12:49.619345012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T08:13:11.555430872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:13:11.55728075Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:13:19.620418637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:13:25.261578561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:57240"
level=debug ts=2026-05-06T08:13:49.477335626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:13:49.622098928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T08:13:50.910796918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:33262"
ts=2026-05-06T08:13:51.985817446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38216"
ts=2026-05-06T08:13:55.263101912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:51226"
ts=2026-05-06T08:14:19.6232828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T08:14:24.998337736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:14:49.62477004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:14:50.913376257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:43894"
ts=2026-05-06T08:14:51.989180725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:40090"
level=debug ts=2026-05-06T08:15:01.66405724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:15:19.626277762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T08:15:21.990174642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:33558"
level=debug ts=2026-05-06T08:15:39.364130998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:15:49.627750574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T08:15:50.916023881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:34758"
ts=2026-05-06T08:16:19.628764669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:16:20.916779906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:33006"
level=debug ts=2026-05-06T08:16:28.383872054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:16:28.385403151Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:16:49.629937385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T08:16:50.918142827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:47878"
level=debug ts=2026-05-06T08:17:09.178924577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:17:19.631004032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T08:17:25.271317602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:53210"
level=debug ts=2026-05-06T08:17:43.93205098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:17:49.632364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T08:17:50.921279542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:49190"
ts=2026-05-06T08:18:19.633345574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:18:25.27446608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:34980"
level=debug ts=2026-05-06T08:18:38.946435942Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:18:49.634958827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
level=debug ts=2026-05-06T08:19:15.633955718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:19:19.636792187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T08:19:49.637837802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:19:52.001925962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:44234"
level=debug ts=2026-05-06T08:20:18.196969784Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:20:18.198502406Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:20:19.639008567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T08:20:22.003473434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:59128"
ts=2026-05-06T08:20:25.278742449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:34948"
level=error ts=2026-05-06T08:20:31.00810408Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:20:31.009893546Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:20:31.012323657Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:20:31.01234596Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:20:49.640831798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:20:50.930650656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:42738"
ts=2026-05-06T08:20:52.005043797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:48968"
level=debug ts=2026-05-06T08:21:04.703389028Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:21:19.642325329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T08:21:48.948132653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:21:49.64326887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:22:19.644910917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
level=debug ts=2026-05-06T08:22:38.04964656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:22:49.646102537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T08:22:50.936975887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:55768"
ts=2026-05-06T08:22:55.287192534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:44102"
ts=2026-05-06T08:23:19.647232962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:23:20.938311154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:44586"
level=debug ts=2026-05-06T08:23:38.431377195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:23:49.64819261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T08:23:50.939933387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:57438"
ts=2026-05-06T08:24:19.649279342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:24:25.291983519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:47670"
level=debug ts=2026-05-06T08:24:34.905179599Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:24:34.906714657Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:24:49.650683542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:24:50.943140518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:52004"
ts=2026-05-06T08:25:19.651470773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
level=debug ts=2026-05-06T08:25:20.352168721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:25:20.944515119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:33594"
ts=2026-05-06T08:25:49.653319748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
level=debug ts=2026-05-06T08:26:09.185646432Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:26:19.654985845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T08:26:49.655743926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T08:26:50.949213097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:49986"
ts=2026-05-06T08:26:55.298663302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:36952"
level=debug ts=2026-05-06T08:26:55.381003141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:27:19.656694469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lcx7p-f6c612bb 10.128.0.83:7946"
ts=2026-05-06T08:27:22.027135226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56148"
ts=2026-05-06T08:27:49.658054101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T08:27:50.95169979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:43426"
level=debug ts=2026-05-06T08:27:51.550564618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:28:19.659742169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T08:28:22.030511235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:42666"
level=debug ts=2026-05-06T08:28:36.94375636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:28:36.945226568Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:28:49.661175142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:29:19.662328323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
level=debug ts=2026-05-06T08:29:28.969799898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:29:49.663923324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07b25c48 10.128.0.85:7946"
ts=2026-05-06T08:29:50.957607055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:49698"
ts=2026-05-06T08:29:52.035136705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:48794"
level=debug ts=2026-05-06T08:30:18.751662799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:30:19.665578289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T08:30:20.958776448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:59892"
level=error ts=2026-05-06T08:30:31.00596223Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:30:31.007651439Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:30:31.009170898Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:30:31.009190749Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:30:49.667075158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w2ksn-89b55c37 10.128.0.84:7946"
ts=2026-05-06T08:30:50.960409586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:42354"
level=debug ts=2026-05-06T08:30:54.37601089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"