level=info ts=2026-04-29T11:44:14.340575357Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-04-29T11:44:14.340643642Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-29T11:44:14.342915419Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-29T11:44:14.3432178Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-29T11:44:14.343289971Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.69 inf=eth0
level=debug ts=2026-04-29T11:44:14.343651117Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-29T11:44:14.343703309Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.69 inf=eth0
level=info ts=2026-04-29T11:44:14.343773111Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-a7f7ac50
ts=2026-04-29T11:44:14.343837093Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-29T11:44:14.343848852Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.69 advertisePort=7946
level=debug ts=2026-04-29T11:44:14.343863401Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.69 advertisePort=7946
level=info ts=2026-04-29T11:44:14.343977451Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-29T11:44:14.350800426Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-29T11:44:14.350817482Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-29T11:44:14.35083119Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.855906ms
level=error ts=2026-04-29T11:44:14.350843225Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-04-29T11:44:14.350852693Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-29T11:44:14.353462807Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T11:44:14.353482905Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-29T11:44:14.357894264Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-29T11:44:14.35795147Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-29T11:44:14.35795226Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-29T11:44:14.357969237Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-29T11:44:14.357986737Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-04-29T11:44:14.357985291Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-29T11:44:14.358060837Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-29T11:44:14.358065542Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-29T11:44:14.35824049Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-29T11:44:14.358248081Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-29T11:44:14.358298886Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-29T11:44:14.358292667Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-29T11:44:14.358310741Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-29T11:44:14.358441613Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-29T11:44:14.358457093Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-29T11:44:14.358477433Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-29T11:44:15.359130902Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-29T11:44:15.544613369Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-29T11:44:15.544671062Z caller=loki.go:615 msg="Loki started" startup_time=1.261752549s
ts=2026-04-29T11:44:16.179982804Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T11:44:16.180010939Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T11:44:18.744893427Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T11:44:18.744919088Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-29T11:44:20.544957884Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-29T11:44:20.54501654Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-29T11:44:24.747329261Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T11:44:24.747350895Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-29T11:44:34.407465246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T11:44:34.643320652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:48954"
ts=2026-04-29T11:44:36.487512244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:46040"
ts=2026-04-29T11:44:37.106193762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:55078"
ts=2026-04-29T11:44:37.106242912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.69:7946"
ts=2026-04-29T11:44:37.107369153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.71:7946"
ts=2026-04-29T11:44:37.109188448Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.70:7946: dial tcp 10.130.0.70:7946: connect: connection refused"
level=info ts=2026-04-29T11:44:37.109198464Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=22.758339559s
ts=2026-04-29T11:44:41.673023065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:58130"
level=debug ts=2026-04-29T11:44:49.500711327Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T11:44:54.187873557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:35588"
ts=2026-04-29T11:44:56.852250519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47060"
ts=2026-04-29T11:45:08.111090872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T11:45:13.331072105Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-29T11:45:37.840683064Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T11:45:38.112936598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T11:46:03.4235981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-29T11:46:03.425253352Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T11:46:08.1143139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T11:46:24.196957582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:35904"
ts=2026-04-29T11:46:27.034509374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:42118"
ts=2026-04-29T11:46:38.116374278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T11:46:43.274566238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T11:47:08.11903304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T11:47:26.864412713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:32774"
ts=2026-04-29T11:47:27.03767144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:49894"
ts=2026-04-29T11:47:38.120733872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
level=debug ts=2026-04-29T11:47:49.943290671Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T11:47:56.866157597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:42070"
ts=2026-04-29T11:47:57.039453053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:43610"
ts=2026-04-29T11:48:08.122620351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T11:48:27.04084815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:50134"
ts=2026-04-29T11:48:38.125120883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T11:49:06.670257682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T11:49:08.127636121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T11:49:24.216189362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:40570"
ts=2026-04-29T11:49:26.871881187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:42408"
ts=2026-04-29T11:49:38.129631357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T11:49:56.874227834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:34152"
ts=2026-04-29T11:50:08.13168718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T11:50:18.566797603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T11:50:26.876428392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:43468"
ts=2026-04-29T11:50:38.134206043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T11:50:57.052207231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:41630"
ts=2026-04-29T11:51:08.137227616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T11:51:24.22784048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:51144"
ts=2026-04-29T11:51:26.880330392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:54242"
level=debug ts=2026-04-29T11:51:31.641550829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-29T11:51:31.643171052Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T11:51:38.140037236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T11:51:54.230512233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:43138"
ts=2026-04-29T11:51:56.882401856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59122"
ts=2026-04-29T11:52:08.14358357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T11:52:25.582991237Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T11:52:38.146833941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T11:52:54.236697234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:55692"
ts=2026-04-29T11:52:56.887099476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:52926"
ts=2026-04-29T11:52:57.060350042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:43334"
ts=2026-04-29T11:53:08.149397328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
level=debug ts=2026-04-29T11:53:14.514084171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T11:53:24.239464399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:45296"
ts=2026-04-29T11:53:38.15057303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T11:54:08.15317539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T11:54:16.746081928Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T11:54:20.545709029Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-29T11:54:20.548311629Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T11:54:20.55017482Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T11:54:20.551894686Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T11:54:20.55191849Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-29T11:54:20.551931169Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-04-29T11:54:24.246449183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:47684"
ts=2026-04-29T11:54:27.066067812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:48696"
ts=2026-04-29T11:54:38.155511231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T11:54:57.067788337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:50586"
ts=2026-04-29T11:55:08.157341087Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T11:55:13.648688153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T11:55:26.900135083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:58384"
ts=2026-04-29T11:55:27.068589155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:51476"
ts=2026-04-29T11:55:38.159368129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
level=debug ts=2026-04-29T11:56:04.652836098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T11:56:04.654731024Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T11:56:08.161078702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T11:56:24.258961295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:54688"
ts=2026-04-29T11:56:27.071736629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:40354"
ts=2026-04-29T11:56:38.162291092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T11:56:54.2624105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:60374"
ts=2026-04-29T11:56:57.072509144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:53562"
level=debug ts=2026-04-29T11:56:59.489302403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T11:57:08.164263355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T11:57:24.265303642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:40102"
ts=2026-04-29T11:57:38.167718352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T11:57:55.450527931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T11:57:57.075217223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:39288"
ts=2026-04-29T11:58:08.170503074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T11:58:36.861757123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T11:58:38.172855011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T11:58:54.275598727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:33076"
ts=2026-04-29T11:58:56.918611272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:33986"
ts=2026-04-29T11:59:08.175863105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T11:59:25.479862294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T11:59:38.179040254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T11:59:57.085234685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:55422"
ts=2026-04-29T12:00:08.181879668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T12:00:22.909712384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T12:00:22.911494027Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:00:38.185215734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:00:54.288263462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:54620"
ts=2026-04-29T12:00:57.088393788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:45710"
level=debug ts=2026-04-29T12:01:04.627543847Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:01:08.187836892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:01:24.290960429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:46892"
ts=2026-04-29T12:01:26.930463494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:56666"
ts=2026-04-29T12:01:27.089826996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:48356"
ts=2026-04-29T12:01:38.190350896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T12:01:49.768237847Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:01:54.294037421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:45328"
ts=2026-04-29T12:02:08.192744318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:02:31.828348978Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:02:38.193804909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:03:08.195529177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T12:03:13.104659465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:03:38.197715596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
level=debug ts=2026-04-29T12:03:49.635152184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T12:03:49.636783175Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:03:54.306931483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:46598"
ts=2026-04-29T12:04:08.198803659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
level=error ts=2026-04-29T12:04:20.557447021Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T12:04:20.559195516Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T12:04:20.560749601Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T12:04:20.560770559Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:04:38.200865755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:04:39.680579074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:04:57.106785678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:55090"
ts=2026-04-29T12:05:08.203245375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:05:24.316479529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:50104"
ts=2026-04-29T12:05:27.108579414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:38820"
ts=2026-04-29T12:05:38.205539064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:05:40.762259877Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:06:08.208228775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:06:26.953742639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:39510"
level=debug ts=2026-04-29T12:06:28.998338145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:06:38.211529695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:06:56.955934569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:38254"
ts=2026-04-29T12:06:57.117162305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:46154"
level=debug ts=2026-04-29T12:07:03.388333893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:07:08.213298048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:07:26.95783938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:39260"
ts=2026-04-29T12:07:38.215095715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:07:39.141555752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T12:07:39.14366486Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:08:08.218336287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:08:24.334703043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:44918"
level=debug ts=2026-04-29T12:08:32.949837778Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:08:38.220442522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:08:57.126924969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:57864"
ts=2026-04-29T12:09:08.222789318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:09:26.967626805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:60328"
level=debug ts=2026-04-29T12:09:31.613807162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:09:38.225793587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:09:56.970583425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:45574"
ts=2026-04-29T12:10:08.228713737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:10:16.625733622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:10:24.346716375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:36992"
ts=2026-04-29T12:10:38.233345121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T12:10:52.582202288Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:11:08.23828129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:11:27.13940162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:57826"
ts=2026-04-29T12:11:38.241358461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:11:53.571422108Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T12:11:53.573316123Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:11:57.140851734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:53068"
ts=2026-04-29T12:12:08.243685496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:12:24.358301557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:36964"
ts=2026-04-29T12:12:38.246042273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T12:12:43.375917044Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:12:54.361681995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:38302"
ts=2026-04-29T12:12:56.985935876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:45356"
ts=2026-04-29T12:12:57.145547673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:44506"
ts=2026-04-29T12:13:08.251678904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T12:13:21.090729993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:13:38.254040412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:13:56.193821033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:14:08.256503722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=error ts=2026-04-29T12:14:20.556559207Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T12:14:20.558542831Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T12:14:20.560337272Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T12:14:20.560360672Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:14:38.258364748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:14:51.903058237Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:15:08.261069378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:15:26.99708583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:60512"
ts=2026-04-29T12:15:27.159979705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:44056"
level=debug ts=2026-04-29T12:15:35.432390594Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T12:15:35.434749402Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:15:38.262741793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:15:56.999456663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:46230"
ts=2026-04-29T12:15:57.160869988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:43736"
ts=2026-04-29T12:16:08.264920962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:16:24.384393564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:39116"
level=debug ts=2026-04-29T12:16:29.436077226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:16:38.268595154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:17:08.271573685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:17:24.39101169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:52058"
ts=2026-04-29T12:17:27.007600379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:34692"
level=debug ts=2026-04-29T12:17:28.21498283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:17:38.274076122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:17:57.010238162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:33716"
ts=2026-04-29T12:18:08.276313613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
level=debug ts=2026-04-29T12:18:13.993806336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:18:38.278973238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:18:54.40091368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:60694"
level=debug ts=2026-04-29T12:18:58.155981335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:19:08.282704383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:19:24.404177254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:47030"
ts=2026-04-29T12:19:38.286003609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:19:54.407658235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:52334"
level=debug ts=2026-04-29T12:19:57.808863722Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T12:19:57.810811106Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:20:08.288593336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:20:24.410644675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:41234"
ts=2026-04-29T12:20:38.290639846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:20:45.039853981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:20:54.413695058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:40216"
ts=2026-04-29T12:20:57.192988459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:52522"
ts=2026-04-29T12:21:08.293226873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:21:24.053294461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:21:38.295176117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:21:54.419193072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:57410"
ts=2026-04-29T12:22:08.29646677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
level=debug ts=2026-04-29T12:22:08.420400036Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:22:27.19981484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:59016"
ts=2026-04-29T12:22:38.298161323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
level=debug ts=2026-04-29T12:22:48.20480008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:22:54.427722247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:54168"
ts=2026-04-29T12:22:57.201185622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:38972"
ts=2026-04-29T12:23:08.300770901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:23:38.303535561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:23:42.23784853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T12:23:42.239634349Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:24:08.305905372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=error ts=2026-04-29T12:24:20.556525444Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T12:24:20.558676792Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T12:24:20.56064564Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T12:24:20.560669702Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T12:24:24.326917356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:24:38.308757904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:24:57.212248164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:33834"
ts=2026-04-29T12:25:08.310963324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
level=debug ts=2026-04-29T12:25:16.979363713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:25:38.312125121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:25:57.21694055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:51836"
level=debug ts=2026-04-29T12:26:07.396095165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:26:08.314465482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:26:38.316828475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:26:57.059284497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:42842"
ts=2026-04-29T12:26:57.221673471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:46758"
level=debug ts=2026-04-29T12:27:04.00344576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:27:08.319475395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:27:38.325289708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:27:54.463293279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:47240"
level=debug ts=2026-04-29T12:28:04.934903514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T12:28:04.936776649Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:28:08.328581237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:28:27.06607413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:52164"
ts=2026-04-29T12:28:38.330528325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:28:54.456124557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:28:57.068037649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47074"
ts=2026-04-29T12:29:08.331839749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:29:24.472693722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:36296"
ts=2026-04-29T12:29:27.070174765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:57002"
ts=2026-04-29T12:29:27.232521184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:41244"
level=debug ts=2026-04-29T12:29:31.237853607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:29:38.334027798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:29:57.234474599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:59888"
level=debug ts=2026-04-29T12:30:07.833405312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:30:08.338637832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:30:24.480130124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:33752"
ts=2026-04-29T12:30:27.236111045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:44696"
ts=2026-04-29T12:30:38.344479132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:30:54.482972661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:53922"
ts=2026-04-29T12:30:57.238821512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:47892"
level=debug ts=2026-04-29T12:31:08.090055936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:31:08.34727848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:31:24.486168128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:42826"
ts=2026-04-29T12:31:38.351010694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:31:57.243892957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:51656"
level=debug ts=2026-04-29T12:31:57.302286412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T12:31:57.304203077Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:32:08.353210536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:32:24.492921776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:37158"
ts=2026-04-29T12:32:27.085310254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:55424"
ts=2026-04-29T12:32:38.354766615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:32:41.69172394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:33:08.356843033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:33:27.090432921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:48378"
level=debug ts=2026-04-29T12:33:37.659209057Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:33:38.358862136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:33:54.501646209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:51998"
ts=2026-04-29T12:34:08.360422548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
level=error ts=2026-04-29T12:34:20.555803334Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T12:34:20.557652717Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T12:34:20.559267615Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T12:34:20.559291553Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:34:27.0958064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:47840"
level=debug ts=2026-04-29T12:34:33.901796317Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:34:38.362359544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:34:57.257402702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:38088"
ts=2026-04-29T12:35:08.364246897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:35:33.049843445Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:35:38.365919017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:35:57.102370523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:55300"
ts=2026-04-29T12:36:08.367480365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:36:27.104821568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:35004"
level=debug ts=2026-04-29T12:36:31.187767042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T12:36:31.190354128Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:36:38.369793021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:36:54.521342076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:40278"
ts=2026-04-29T12:36:57.266186722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:59922"
ts=2026-04-29T12:37:08.37268991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:37:23.853991058Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:37:27.111733899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:58710"
ts=2026-04-29T12:37:38.374556099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:37:54.528123886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:37582"
ts=2026-04-29T12:37:57.269711521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:53258"
ts=2026-04-29T12:38:08.377169599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T12:38:19.359518409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:38:24.531593644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:39654"
ts=2026-04-29T12:38:27.120965118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:52090"
ts=2026-04-29T12:38:27.270896234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:43528"
ts=2026-04-29T12:38:38.379685782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:38:54.535339463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:60798"
ts=2026-04-29T12:39:08.381754963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:39:18.930520154Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:39:24.538568641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:35744"
ts=2026-04-29T12:39:38.384285062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:39:54.541089284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:46740"
level=debug ts=2026-04-29T12:39:55.513760042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:40:08.385914228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:40:24.544563428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:39650"
ts=2026-04-29T12:40:27.279783659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:57784"
level=debug ts=2026-04-29T12:40:31.672999741Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T12:40:31.674490519Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:40:38.386955537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:40:57.280620523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:53966"
ts=2026-04-29T12:41:08.388522539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
level=debug ts=2026-04-29T12:41:21.809340151Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:41:27.13529983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:38252"
ts=2026-04-29T12:41:38.390528226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:42:08.394065701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T12:42:19.446526533Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:42:38.39647362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:42:54.560237075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:51828"
level=debug ts=2026-04-29T12:43:04.965359378Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:43:08.399026815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:43:38.400713092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
level=debug ts=2026-04-29T12:43:49.400132918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:43:57.297832586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:44890"
ts=2026-04-29T12:44:08.402662926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=error ts=2026-04-29T12:44:20.555468458Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T12:44:20.557330628Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T12:44:20.558998933Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T12:44:20.559024112Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:44:38.405794881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T12:44:43.867214972Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T12:44:43.868932196Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:44:57.152087356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:51480"
ts=2026-04-29T12:45:08.409204639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T12:45:24.710975675Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:45:27.304769665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:47454"
ts=2026-04-29T12:45:38.412698029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:46:08.415568839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:46:24.579158291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:42926"
level=debug ts=2026-04-29T12:46:26.107168062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:46:38.41758481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:46:57.16238754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:56266"
ts=2026-04-29T12:47:08.419804016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:47:13.90349979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:47:27.164516757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:37178"
ts=2026-04-29T12:47:38.421113797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:47:57.316477333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:53116"
level=debug ts=2026-04-29T12:47:58.649258026Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:48:08.422383675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:48:24.59223609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:48044"
ts=2026-04-29T12:48:27.168655105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:53756"
ts=2026-04-29T12:48:38.424981251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T12:48:40.759244343Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T12:48:40.760793488Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:49:08.427324062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:49:27.322396137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:37880"
level=debug ts=2026-04-29T12:49:27.924655618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:49:38.429081362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:50:08.430644478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:50:22.331378452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:50:24.603442739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:59218"
ts=2026-04-29T12:50:38.432161837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
ts=2026-04-29T12:50:57.180900164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:33104"
level=debug ts=2026-04-29T12:50:58.916187828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:51:08.433675994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
ts=2026-04-29T12:51:27.331055408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:59690"
ts=2026-04-29T12:51:38.436070343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-kstr8-8f61b872 10.130.0.71:7946"
level=debug ts=2026-04-29T12:51:43.955651238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T12:52:08.438553078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
level=debug ts=2026-04-29T12:52:24.141538828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T12:52:24.143089479Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T12:52:24.613502529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:54714"
ts=2026-04-29T12:52:38.442027715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8lx8t-5b5dae36 10.129.0.96:7946"
ts=2026-04-29T12:52:54.616368047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.96:49482"
ts=2026-04-29T12:52:57.190095562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:52320"
ts=2026-04-29T12:52:57.337429303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:52836"
ts=2026-04-29T12:53:08.444215188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118171df 10.128.0.70:7946"
level=debug ts=2026-04-29T12:53:13.681435302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"