level=info ts=2026-05-07T08:18:47.5411789Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-07T08:18:47.54123351Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-07T08:18:47.542780205Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-07T08:18:47.543144386Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T08:18:47.543249457Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.71 inf=eth0
level=debug ts=2026-05-07T08:18:47.543814054Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T08:18:47.543885747Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.71 inf=eth0
level=info ts=2026-05-07T08:18:47.544401454Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-1169009b
ts=2026-05-07T08:18:47.544473742Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-07T08:18:47.544491257Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.71 advertisePort=7946
level=debug ts=2026-05-07T08:18:47.544522963Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.71 advertisePort=7946
level=info ts=2026-05-07T08:18:47.544647456Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-07T08:18:47.552888947Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T08:18:47.552911714Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T08:18:47.552926855Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=8.281617ms
level=error ts=2026-05-07T08:18:47.552938352Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-07T08:18:47.552949111Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-07T08:18:47.55837623Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T08:18:47.558396847Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-07T08:18:47.563979932Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-07T08:18:47.563991469Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-07T08:18:47.564005489Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-07T08:18:47.564090064Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-07T08:18:47.56410649Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-07T08:18:47.564105955Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-07T08:18:47.564120665Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-07T08:18:47.564210845Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-07T08:18:47.564226464Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-05-07T08:18:47.56439558Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-07T08:18:47.564443446Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-07T08:18:47.564471584Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-07T08:18:47.564486554Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-07T08:18:47.564588121Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-07T08:18:47.56459644Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-07T08:18:47.564607195Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-07T08:18:48.565331683Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-07T08:18:48.704287945Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-07T08:18:48.704347607Z caller=loki.go:615 msg="Loki started" startup_time=1.242382441s
ts=2026-05-07T08:18:48.8738259Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T08:18:48.873850982Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T08:18:51.412125303Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T08:18:51.412150028Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-07T08:18:53.70492102Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-07T08:18:53.704983501Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-07T08:18:58.381345868Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T08:18:58.381368782Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-07T08:19:01.213452753Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T08:19:09.257016079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:60448"
ts=2026-05-07T08:19:11.26077086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40264"
ts=2026-05-07T08:19:11.56957386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.69:7946"
ts=2026-05-07T08:19:11.570136618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.72:7946"
ts=2026-05-07T08:19:11.571288434Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.73:7946: dial tcp 10.130.0.73:7946: connect: connection refused"
ts=2026-05-07T08:19:11.572166053Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.66:7946: dial tcp 10.130.0.66:7946: connect: connection refused"
ts=2026-05-07T08:19:11.572942257Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.70:7946: dial tcp 10.130.0.70:7946: connect: connection refused"
ts=2026-05-07T08:19:11.573051583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.71:7946"
ts=2026-05-07T08:19:11.573071719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:48820"
level=info ts=2026-05-07T08:19:11.573338323Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=24.020382433s
ts=2026-05-07T08:19:19.530630028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40324"
level=debug ts=2026-05-07T08:19:25.125790381Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T08:19:27.523933463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:52738"
ts=2026-05-07T08:19:45.440729003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:19:48.440528752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:41576"
level=debug ts=2026-05-07T08:19:49.134105748Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T08:19:49.532202153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:50118"
level=debug ts=2026-05-07T08:20:12.715811818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T08:20:15.442749973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
level=debug ts=2026-05-07T08:20:36.419847933Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T08:20:36.421611963Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T08:20:45.444114139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:20:49.53545804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45308"
ts=2026-05-07T08:21:15.445960093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:21:16.60287248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:40974"
level=debug ts=2026-05-07T08:21:16.603808605Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T08:21:45.448104862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:21:46.604037513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42920"
level=debug ts=2026-05-07T08:22:06.386526989Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T08:22:15.449586186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:22:45.451493172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:22:48.454575098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:51326"
ts=2026-05-07T08:23:15.452861295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:23:18.456351251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:57634"
level=debug ts=2026-05-07T08:23:19.105743774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T08:23:45.45436206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:23:49.546374006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58660"
ts=2026-05-07T08:24:15.4562944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:24:16.611456425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43230"
ts=2026-05-07T08:24:18.459704394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:46236"
ts=2026-05-07T08:24:19.547930451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41670"
level=debug ts=2026-05-07T08:24:34.019054302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T08:24:45.457877279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:24:49.5493052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51198"
ts=2026-05-07T08:25:15.458893725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:25:16.614595501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41938"
ts=2026-05-07T08:25:18.464711471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:32780"
ts=2026-05-07T08:25:19.550332014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33308"
level=debug ts=2026-05-07T08:25:35.884178369Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T08:25:35.885781203Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T08:25:45.460315496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:25:48.466926704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:34302"
ts=2026-05-07T08:25:49.551726412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:48720"
ts=2026-05-07T08:26:15.461416977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:26:16.617017522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43596"
level=debug ts=2026-05-07T08:26:28.868354441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:26:45.463769761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:26:48.471365428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:40618"
ts=2026-05-07T08:27:15.465501515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:27:16.62012175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:33056"
ts=2026-05-07T08:27:19.556298847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:44696"
level=debug ts=2026-05-07T08:27:21.49984405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:27:45.46627804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:27:46.621449563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:34344"
ts=2026-05-07T08:27:48.475962716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:54782"
level=debug ts=2026-05-07T08:28:14.382931045Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:28:15.467963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:28:45.469585887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:28:49.560892794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34526"
level=info ts=2026-05-07T08:28:53.705098368Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-07T08:28:53.708918264Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T08:28:53.710824871Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T08:28:53.71252748Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T08:28:53.712555053Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T08:28:53.712567756Z caller=tables_manager.go:148 msg="compactor started"
level=debug ts=2026-05-07T08:29:02.524854815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:29:15.47163088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:29:45.472783723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
level=debug ts=2026-05-07T08:29:46.898906563Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T08:29:46.900556301Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T08:30:15.47391029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:30:45.475750524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
level=debug ts=2026-05-07T08:30:47.412266951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:30:48.488813234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:55926"
ts=2026-05-07T08:30:49.568177572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:55890"
ts=2026-05-07T08:31:15.477279932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:31:16.632633648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:40808"
level=debug ts=2026-05-07T08:31:41.865475899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:31:45.478230475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:32:15.479764481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
level=debug ts=2026-05-07T08:32:28.711515586Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:32:45.48164255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
level=debug ts=2026-05-07T08:33:06.15483959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:33:15.483270977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:33:16.639421054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:45218"
ts=2026-05-07T08:33:19.574762092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39752"
level=debug ts=2026-05-07T08:33:44.488716619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T08:33:44.493270647Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T08:33:45.484481933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:34:15.485846977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
level=debug ts=2026-05-07T08:34:33.699059003Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:34:45.487651222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:34:49.578136846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58934"
ts=2026-05-07T08:35:15.490363785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
level=debug ts=2026-05-07T08:35:24.550388524Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:35:45.492986727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:35:46.647378562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38236"
ts=2026-05-07T08:35:49.581944397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59266"
level=debug ts=2026-05-07T08:36:02.801693192Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:36:15.495519856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:36:19.582922831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51594"
level=debug ts=2026-05-07T08:36:41.412942175Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:36:45.49728669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:36:48.513862499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:59796"
ts=2026-05-07T08:37:15.498740153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:37:19.586280286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52656"
level=debug ts=2026-05-07T08:37:35.021017635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T08:37:35.023114765Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T08:37:45.504554046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:37:48.519199982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:51060"
ts=2026-05-07T08:38:15.506646702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
level=debug ts=2026-05-07T08:38:22.923715214Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:38:45.509231022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:38:46.658140756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53052"
level=error ts=2026-05-07T08:38:53.717972125Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T08:38:53.719826537Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T08:38:53.721416502Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T08:38:53.721441994Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T08:39:09.298737689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:39:15.510911156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:39:18.524894633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:46470"
ts=2026-05-07T08:39:45.514556702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:39:46.662065895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:46502"
ts=2026-05-07T08:39:48.528810069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:38324"
level=debug ts=2026-05-07T08:39:49.945744442Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:40:15.5193658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:40:18.531099855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:53220"
level=debug ts=2026-05-07T08:40:31.816544505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:40:45.520538446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:40:46.665531164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:39512"
ts=2026-05-07T08:40:49.600156756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36880"
ts=2026-05-07T08:41:15.522424133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
level=debug ts=2026-05-07T08:41:24.572872232Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T08:41:24.574718597Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T08:41:45.524921051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:41:46.668333738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:40048"
ts=2026-05-07T08:42:15.527102426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
level=debug ts=2026-05-07T08:42:26.716020602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:42:45.529382211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:42:48.543033986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:57106"
level=debug ts=2026-05-07T08:43:05.184985202Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:43:15.530654601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:43:45.534483355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:43:46.674824423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43362"
ts=2026-05-07T08:43:48.548048628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:38500"
level=debug ts=2026-05-07T08:43:49.032047262Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:44:15.536016406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
level=debug ts=2026-05-07T08:44:44.462587789Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:44:45.537497594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:44:46.677670022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:50342"
ts=2026-05-07T08:44:48.55183386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:38394"
ts=2026-05-07T08:45:15.539009674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:45:18.553541552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:36492"
level=debug ts=2026-05-07T08:45:18.939784612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T08:45:18.943863183Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T08:45:19.621338266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43344"
ts=2026-05-07T08:45:45.540764487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
level=debug ts=2026-05-07T08:46:04.38191178Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:46:15.54236998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:46:18.557762622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:38180"
ts=2026-05-07T08:46:19.624345377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:35132"
ts=2026-05-07T08:46:45.543830574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:46:48.559951771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:45748"
level=debug ts=2026-05-07T08:46:58.294003573Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:47:15.545532098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
level=debug ts=2026-05-07T08:47:37.035769888Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:47:45.547775204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:47:46.689201782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41842"
ts=2026-05-07T08:48:15.549859124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:48:18.567231051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:55420"
ts=2026-05-07T08:48:19.632682149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41390"
level=debug ts=2026-05-07T08:48:24.632952082Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:48:45.551821622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:48:46.69333342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:57298"
ts=2026-05-07T08:48:48.570041925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:49106"
level=error ts=2026-05-07T08:48:53.716314743Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T08:48:53.717964847Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T08:48:53.71951731Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T08:48:53.719539105Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T08:48:58.723839593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T08:48:58.725597793Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T08:49:15.553021569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:49:19.635479448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36486"
ts=2026-05-07T08:49:45.554825848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
level=debug ts=2026-05-07T08:49:45.682036965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:49:49.637504065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45630"
ts=2026-05-07T08:50:15.556415461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:50:16.697211948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52182"
ts=2026-05-07T08:50:45.557717413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
level=debug ts=2026-05-07T08:50:45.835032744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:50:48.578722207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:38086"
ts=2026-05-07T08:50:49.641113214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:56038"
ts=2026-05-07T08:51:15.559293019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
level=debug ts=2026-05-07T08:51:20.74378542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:51:45.56104829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:51:46.701741719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42760"
ts=2026-05-07T08:51:48.582941728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:45788"
level=debug ts=2026-05-07T08:52:11.931443321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:52:15.562168324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:52:16.702947517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:40726"
ts=2026-05-07T08:52:18.58538652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:55306"
ts=2026-05-07T08:52:45.563379311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:52:49.64734477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42198"
level=debug ts=2026-05-07T08:52:52.154021608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T08:52:52.155927018Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T08:53:15.5654841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
level=debug ts=2026-05-07T08:53:35.124544756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:53:45.567287584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:53:46.708218237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:58812"
ts=2026-05-07T08:53:49.650767995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47444"
ts=2026-05-07T08:54:15.568692007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
level=debug ts=2026-05-07T08:54:33.330857971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:54:45.569723393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:54:46.711716933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41508"
ts=2026-05-07T08:55:15.570968708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:55:16.713128652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53406"
level=debug ts=2026-05-07T08:55:32.601051937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:55:45.571992615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:55:49.657982502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:50448"
ts=2026-05-07T08:56:15.573432578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
level=debug ts=2026-05-07T08:56:35.189928231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:56:45.574659361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T08:56:46.718174768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:57522"
ts=2026-05-07T08:56:48.608552751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:50862"
ts=2026-05-07T08:56:49.661462394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36354"
ts=2026-05-07T08:57:15.575478327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T08:57:16.719855735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38736"
level=debug ts=2026-05-07T08:57:19.065162913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T08:57:19.066837572Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T08:57:45.576966182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:57:48.614064424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:53712"
level=debug ts=2026-05-07T08:58:03.359619619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:58:15.579004589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
level=debug ts=2026-05-07T08:58:38.731234376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:58:45.580950519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:58:46.724578062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37670"
ts=2026-05-07T08:58:48.635888139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:53218"
level=error ts=2026-05-07T08:58:53.716276786Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T08:58:53.717958263Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T08:58:53.719624875Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T08:58:53.719645381Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T08:59:15.582458316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
level=debug ts=2026-05-07T08:59:17.742929209Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T08:59:45.584402095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T08:59:49.671595238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:57314"
level=debug ts=2026-05-07T09:00:00.858591932Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T09:00:15.585869613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T09:00:18.643942227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:36228"
ts=2026-05-07T09:00:45.587275332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T09:00:48.650445684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:40024"
ts=2026-05-07T09:00:49.674277442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42524"
level=debug ts=2026-05-07T09:00:54.463008397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T09:00:54.464733681Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T09:01:15.589447215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T09:01:19.675768797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58394"
level=debug ts=2026-05-07T09:01:35.200865338Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T09:01:45.590980711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T09:01:48.654892408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:49224"
ts=2026-05-07T09:01:49.6773342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:55344"
level=debug ts=2026-05-07T09:02:11.667451895Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T09:02:15.592296906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T09:02:45.594414602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T09:02:46.73974921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41396"
level=debug ts=2026-05-07T09:03:08.801548533Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T09:03:15.596608329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T09:03:16.741492135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52196"
ts=2026-05-07T09:03:18.660894772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:33722"
ts=2026-05-07T09:03:45.598456014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T09:03:48.662906715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:56598"
level=debug ts=2026-05-07T09:03:54.530229998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T09:04:15.600202398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T09:04:19.686488136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:48466"
ts=2026-05-07T09:04:45.601176764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T09:04:46.746011195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:35218"
level=debug ts=2026-05-07T09:04:51.420694068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T09:04:51.4223387Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T09:05:15.603151255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
level=debug ts=2026-05-07T09:05:43.769242658Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T09:05:45.604816428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T09:05:48.679330301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:52256"
ts=2026-05-07T09:06:15.607059602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T09:06:18.681845116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:58090"
level=debug ts=2026-05-07T09:06:34.860119519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T09:06:45.608861226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T09:06:48.684018772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:47002"
ts=2026-05-07T09:06:49.695894831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51346"
ts=2026-05-07T09:07:15.610226332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T09:07:16.754436127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:54604"
ts=2026-05-07T09:07:18.686283064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:43898"
level=debug ts=2026-05-07T09:07:26.728212329Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T09:07:45.612035133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T09:08:15.616049449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T09:08:18.691990316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:57648"
level=debug ts=2026-05-07T09:08:28.788776506Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T09:08:45.61925898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
level=error ts=2026-05-07T09:08:53.716299283Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T09:08:53.718173743Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T09:08:53.719898703Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T09:08:53.71992419Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T09:09:04.383551924Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T09:09:04.385207578Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T09:09:15.623155034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T09:09:16.760019965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42504"
ts=2026-05-07T09:09:19.703735136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:57788"
ts=2026-05-07T09:09:45.624458409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-tkxkm-e2a244da 10.130.0.72:7946"
ts=2026-05-07T09:09:49.705532385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58194"
level=debug ts=2026-05-07T09:09:50.315940344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T09:10:15.626087955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T09:10:18.700819889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:54752"
level=debug ts=2026-05-07T09:10:27.020490172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T09:10:45.627259879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T09:10:46.765960621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:58342"
ts=2026-05-07T09:10:48.70297656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:54488"
level=debug ts=2026-05-07T09:11:04.206989815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T09:11:15.628234312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T09:11:45.63005015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-xl97t-196fb049 10.130.0.69:7946"
ts=2026-05-07T09:11:48.70637533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:33510"
ts=2026-05-07T09:11:49.712996813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:48478"
level=debug ts=2026-05-07T09:11:59.54612008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T09:12:15.632158246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-32cca362 10.129.0.77:7946"
ts=2026-05-07T09:12:16.77165469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:54140"
ts=2026-05-07T09:12:18.708110882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:47988"
ts=2026-05-07T09:12:19.714173831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40302"