level=info ts=2026-05-06T06:57:14.178878025Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-06T06:57:14.178914882Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-06T06:57:14.180925772Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-06T06:57:14.181228754Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-06T06:57:14.181300855Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.86 inf=eth0
level=debug ts=2026-05-06T06:57:14.181719538Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-06T06:57:14.181778519Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.86 inf=eth0
level=info ts=2026-05-06T06:57:14.181848367Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-014b5263
ts=2026-05-06T06:57:14.181917782Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-06T06:57:14.181928633Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.86 advertisePort=7946
level=debug ts=2026-05-06T06:57:14.181945221Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.86 advertisePort=7946
level=info ts=2026-05-06T06:57:14.18205356Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-06T06:57:14.188469239Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-06T06:57:14.188493982Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-06T06:57:14.188504752Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.453282ms
level=error ts=2026-05-06T06:57:14.188514619Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-06T06:57:14.188526837Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-06T06:57:14.190859167Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T06:57:14.190876169Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-06T06:57:14.195460059Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-06T06:57:14.195453959Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-06T06:57:14.195483512Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-06T06:57:14.195477008Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-06T06:57:14.195500346Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-05-06T06:57:14.195497958Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-06T06:57:14.195574554Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-06T06:57:14.195578758Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-06T06:57:14.19578571Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-06T06:57:14.195790035Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-06T06:57:14.195822688Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-06T06:57:14.195815823Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-06T06:57:14.19584118Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-06T06:57:14.195971441Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-06T06:57:14.195997235Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-06T06:57:14.196022523Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-06T06:57:15.196779263Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-06T06:57:15.369150771Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-06T06:57:15.369203128Z caller=loki.go:615 msg="Loki started" startup_time=1.250279572s
ts=2026-05-06T06:57:15.57469129Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T06:57:15.574716704Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:57:18.576375166Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T06:57:18.576407016Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-06T06:57:20.369709264Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-06T06:57:20.369778561Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-06T06:57:24.432335105Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T06:57:24.43235842Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-06T06:57:33.010609341Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:57:36.381227063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43108"
ts=2026-05-06T06:57:36.3812852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.86:7946"
ts=2026-05-06T06:57:36.383590272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.57:7946"
ts=2026-05-06T06:57:36.386612044Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.58:7946: dial tcp 10.130.0.58:7946: connect: connection refused"
level=info ts=2026-05-06T06:57:36.386631172Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=22.198099106s
ts=2026-05-06T06:57:38.761426502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:41342"
ts=2026-05-06T06:57:38.963482551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:37132"
ts=2026-05-06T06:57:42.147985643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:51212"
level=debug ts=2026-05-06T06:57:49.94557864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:58:07.031856019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:40840"
level=debug ts=2026-05-06T06:58:09.442427064Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:58:11.001508028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T06:58:32.80478336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T06:58:34.977787245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:49618"
ts=2026-05-06T06:58:41.003573706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T06:59:04.76824709Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-06T06:59:04.770235767Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T06:59:11.006504136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T06:59:34.98361747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:39328"
ts=2026-05-06T06:59:37.039454046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:35176"
ts=2026-05-06T06:59:41.010097507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T06:59:43.767500741Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T07:00:07.041681635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:40348"
ts=2026-05-06T07:00:11.012619629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:00:34.989262151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:43466"
level=debug ts=2026-05-06T07:00:39.621613639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T07:00:41.014567835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:01:07.044991922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:52484"
ts=2026-05-06T07:01:11.016773757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:01:41.019748761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:01:41.439446756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T07:01:54.392851015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:59410"
ts=2026-05-06T07:02:07.048037011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:44648"
ts=2026-05-06T07:02:11.022733317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T07:02:37.049759266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:43162"
ts=2026-05-06T07:02:41.025078912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:02:45.395895137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T07:03:05.005339316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:33038"
ts=2026-05-06T07:03:11.027394121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:03:35.008226654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:33366"
ts=2026-05-06T07:03:41.029933909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:03:47.009368091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-06T07:03:47.011011522Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:03:54.405965612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:36860"
ts=2026-05-06T07:04:07.058876123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53178"
ts=2026-05-06T07:04:11.032756564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:04:24.407603998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:40888"
level=debug ts=2026-05-06T07:04:24.854311085Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:04:37.061302076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:33290"
ts=2026-05-06T07:04:41.034896209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T07:04:54.409622577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:40028"
ts=2026-05-06T07:05:05.017796904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:53724"
ts=2026-05-06T07:05:11.037935761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T07:05:13.074796079Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:05:41.039652836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T07:05:57.66182266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:06:11.041936526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:06:24.418403197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:34674"
ts=2026-05-06T07:06:37.069919451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:40926"
ts=2026-05-06T07:06:41.043970132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:06:57.620677714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:07:05.030076229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:57762"
ts=2026-05-06T07:07:11.046440478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=info ts=2026-05-06T07:07:20.370545847Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-06T07:07:20.37432342Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:07:20.376142045Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:07:20.378045045Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:07:20.378076248Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-06T07:07:20.378089019Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-06T07:07:37.072911615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:43098"
ts=2026-05-06T07:07:41.049101254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:07:50.434027132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:07:50.435953473Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:07:54.426894726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:45330"
ts=2026-05-06T07:08:05.03585915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:39324"
ts=2026-05-06T07:08:11.051277215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:08:24.428695016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:51418"
ts=2026-05-06T07:08:35.038989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:45376"
ts=2026-05-06T07:08:41.053968397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:08:42.15196295Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:08:54.430823174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:52324"
ts=2026-05-06T07:09:07.080768506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:46196"
ts=2026-05-06T07:09:11.056683771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T07:09:36.275093754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:09:37.082360962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:41944"
ts=2026-05-06T07:09:41.059025376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T07:10:11.061365418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:10:24.739711099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:10:41.064153037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T07:11:07.086613388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:47272"
ts=2026-05-06T07:11:11.067037556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:11:13.461019461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:11:37.088871964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:59190"
ts=2026-05-06T07:11:41.068842102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:11:54.445294934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:51342"
level=debug ts=2026-05-06T07:12:01.905316692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:12:01.906912646Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:12:05.060278999Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:42006"
ts=2026-05-06T07:12:11.070343858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:12:24.44782316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:52644"
ts=2026-05-06T07:12:41.072536147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:12:43.089786532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:13:07.095487743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:39018"
ts=2026-05-06T07:13:11.075261522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:13:24.450626974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:44484"
level=debug ts=2026-05-06T07:13:36.787568272Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:13:41.077849229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T07:14:05.073194704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:48890"
ts=2026-05-06T07:14:11.080389164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:14:24.455412712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:54452"
ts=2026-05-06T07:14:35.076495334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:40888"
level=debug ts=2026-05-06T07:14:35.969087818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:14:37.101861514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:51902"
ts=2026-05-06T07:14:41.082387026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:15:11.085011223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:15:18.124246836Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:15:41.08722922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:15:54.462288686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:53526"
ts=2026-05-06T07:16:05.086621253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:35542"
ts=2026-05-06T07:16:11.089696825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T07:16:19.990915315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:16:19.993039615Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:16:35.089826257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:48712"
ts=2026-05-06T07:16:37.112117663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:38208"
ts=2026-05-06T07:16:41.09209308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T07:16:54.466298966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:50970"
ts=2026-05-06T07:17:07.11400797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53942"
level=debug ts=2026-05-06T07:17:08.008695623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:17:11.094701151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=error ts=2026-05-06T07:17:20.384171865Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:17:20.387050861Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:17:20.389845983Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:17:20.389886103Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:17:37.115854708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:34280"
ts=2026-05-06T07:17:41.097148086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T07:17:54.471655155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:38032"
level=debug ts=2026-05-06T07:18:04.546393627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:18:11.10060775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T07:18:41.103418112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:18:45.679063577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:18:54.478069368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:58542"
ts=2026-05-06T07:19:11.106485315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T07:19:34.813008849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:19:41.108812015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T07:20:07.127117076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:52100"
ts=2026-05-06T07:20:11.112327788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:20:32.51921068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:20:32.521063988Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:20:35.115095558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:51768"
ts=2026-05-06T07:20:37.129211213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:49890"
ts=2026-05-06T07:20:41.114826299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:21:05.118685402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:45644"
ts=2026-05-06T07:21:11.117182945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:21:16.536918444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:21:41.120200527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:21:54.491812578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:41062"
level=debug ts=2026-05-06T07:22:09.003080375Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:22:11.122719896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:22:24.494145873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:38164"
ts=2026-05-06T07:22:41.1244732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:22:55.329171443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:23:11.126489482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:23:41.128849459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:23:51.064423834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:23:54.50195802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:41426"
ts=2026-05-06T07:24:07.143820324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:51210"
ts=2026-05-06T07:24:11.135204419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T07:24:41.140829439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:24:41.193485426Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:24:41.195391171Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:25:11.143478214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:25:35.81577429Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:25:41.145293291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:26:11.149132073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:26:19.637845552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:26:35.160594879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:35764"
ts=2026-05-06T07:26:41.151419666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:27:10.820946557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:27:11.154047912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=error ts=2026-05-06T07:27:20.381865934Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:27:20.384810751Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:27:20.387481242Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:27:20.387527941Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:27:24.522581255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:60696"
ts=2026-05-06T07:27:35.168994678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:59744"
ts=2026-05-06T07:27:41.156849404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T07:27:49.561888115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:27:54.524482249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:45424"
ts=2026-05-06T07:28:07.174013196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53244"
ts=2026-05-06T07:28:11.158947115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:28:41.161413352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:28:44.24581109Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:28:44.247973138Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:28:54.530254079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:35678"
ts=2026-05-06T07:29:05.179343031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:48638"
ts=2026-05-06T07:29:11.169829118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:29:34.851456693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:29:41.186928735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:29:54.534695242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:57984"
ts=2026-05-06T07:30:11.189269587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:30:29.088973449Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:30:37.182518574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:59746"
ts=2026-05-06T07:30:41.190873112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:31:05.191760234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:60526"
ts=2026-05-06T07:31:11.192691209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:31:22.169967593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:31:24.543062657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:56204"
ts=2026-05-06T07:31:41.19524679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:31:54.545563847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:41390"
level=debug ts=2026-05-06T07:32:10.849795685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:32:11.19724589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:32:24.547335012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:58088"
ts=2026-05-06T07:32:41.198791468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:33:05.203840577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:53588"
ts=2026-05-06T07:33:07.191171318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53138"
ts=2026-05-06T07:33:11.200742951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T07:33:11.573513056Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:33:11.575510114Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:33:35.206691719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:52896"
ts=2026-05-06T07:33:41.202903123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T07:33:49.701913334Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:34:05.209387108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:41926"
ts=2026-05-06T07:34:11.205431795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:34:35.490551829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:34:41.20792952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:35:11.21015788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:35:20.997700788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:35:24.557709369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:50566"
ts=2026-05-06T07:35:37.200825413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:41744"
ts=2026-05-06T07:35:41.212582323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:35:58.876519552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:36:05.221095885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:52224"
ts=2026-05-06T07:36:07.202419451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:34474"
ts=2026-05-06T07:36:11.215080738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T07:36:24.566887007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:53316"
ts=2026-05-06T07:36:37.204466798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:38622"
ts=2026-05-06T07:36:41.217379168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:36:54.201547959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:36:54.203197497Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:37:11.219027188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=error ts=2026-05-06T07:37:20.383404085Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:37:20.385549224Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:37:20.387557302Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:37:20.387588753Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-06T07:37:32.242781092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:37:35.230342667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:37270"
ts=2026-05-06T07:37:37.207867699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:41734"
ts=2026-05-06T07:37:41.221304149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:37:54.573203665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:50084"
ts=2026-05-06T07:38:05.232885552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:33986"
level=debug ts=2026-05-06T07:38:09.301294169Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:38:11.222933163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:38:37.211974953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:38838"
ts=2026-05-06T07:38:41.225563423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:38:59.361205719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:39:05.23942839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:54602"
ts=2026-05-06T07:39:07.214494693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:49874"
ts=2026-05-06T07:39:11.228269842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:39:41.230401856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T07:39:49.977649188Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:40:05.245283078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:53924"
ts=2026-05-06T07:40:11.232636371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:40:24.581002786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:60822"
ts=2026-05-06T07:40:35.248826041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:33546"
ts=2026-05-06T07:40:41.235024093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:40:43.949057024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:40:43.95124139Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:41:05.251955262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:40516"
ts=2026-05-06T07:41:07.224608286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:39194"
ts=2026-05-06T07:41:11.237256887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:41:40.806944805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:41:41.239709183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:42:05.258327778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:39834"
ts=2026-05-06T07:42:11.242140555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:42:33.470932127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:42:35.261522862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:41616"
ts=2026-05-06T07:42:41.244880503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T07:43:05.264014469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:35416"
ts=2026-05-06T07:43:11.248413797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:43:12.235295457Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:43:35.266730989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:59822"
ts=2026-05-06T07:43:37.234864306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53510"
ts=2026-05-06T07:43:41.250843265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:43:54.59986883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:36446"
level=debug ts=2026-05-06T07:43:54.903217737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:44:05.269913299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:60850"
ts=2026-05-06T07:44:11.252343872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:44:37.238915957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:58482"
level=debug ts=2026-05-06T07:44:39.253598157Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:44:39.255383862Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:44:41.254546565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:45:11.256500667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T07:45:18.180061971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:45:37.243840644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:35160"
ts=2026-05-06T07:45:41.259369373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:45:53.143205171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:46:05.282714068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:52828"
ts=2026-05-06T07:46:11.261621618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T07:46:31.979444975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:46:35.285292864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:49854"
ts=2026-05-06T07:46:37.249515254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:43006"
ts=2026-05-06T07:46:41.263498221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:46:54.618025856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:34466"
ts=2026-05-06T07:47:05.288335927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:48696"
ts=2026-05-06T07:47:11.265832555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=error ts=2026-05-06T07:47:20.381564761Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:47:20.383578392Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:47:20.38546129Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:47:20.385488634Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-06T07:47:22.256007147Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:47:35.291172356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:51442"
ts=2026-05-06T07:47:41.26833619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:48:04.072206896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:48:04.074026828Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:48:11.298387915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T07:48:35.297536894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:35188"
level=debug ts=2026-05-06T07:48:40.792980834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:48:41.301338707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T07:49:05.300857656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:56586"
ts=2026-05-06T07:49:11.303465882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:49:41.306365713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:49:41.459189328Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:50:11.308886541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T07:50:28.494860424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:50:41.310820671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:51:05.312818947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:60758"
ts=2026-05-06T07:51:11.313052446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:51:21.751112922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:51:37.275477924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:60022"
ts=2026-05-06T07:51:41.316073868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T07:52:01.471933814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:52:01.473457506Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:52:11.31775075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:52:24.640397488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:33468"
ts=2026-05-06T07:52:37.279869432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:38418"
ts=2026-05-06T07:52:41.319699418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T07:52:43.705691781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:52:54.642098539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:57996"
ts=2026-05-06T07:53:07.282233649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:35096"
ts=2026-05-06T07:53:11.321750816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:53:25.297810066Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:53:41.323874659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:53:54.644517049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:55328"
ts=2026-05-06T07:54:05.330504316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:36188"
ts=2026-05-06T07:54:07.286961144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:47262"
ts=2026-05-06T07:54:11.325418714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:54:24.64703764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:44920"
level=debug ts=2026-05-06T07:54:25.096355148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:54:37.289317192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:59526"
ts=2026-05-06T07:54:41.327333564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T07:54:54.648568717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:41176"
ts=2026-05-06T07:55:07.291212909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:50128"
ts=2026-05-06T07:55:11.329639224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T07:55:23.505638525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:55:35.339930765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:48362"
ts=2026-05-06T07:55:41.331633881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:56:05.342413985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:45714"
ts=2026-05-06T07:56:11.33412626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:56:22.395898579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:56:22.397877463Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:56:24.65497194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:42750"
ts=2026-05-06T07:56:35.345763748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:52468"
ts=2026-05-06T07:56:41.33708464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T07:56:54.65690295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:54806"
ts=2026-05-06T07:57:11.33921968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:57:12.485959907Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T07:57:20.381722166Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:57:20.383714776Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:57:20.385575135Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T07:57:20.385599335Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T07:57:41.341553901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T07:58:02.724878051Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:58:07.306201216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:40064"
ts=2026-05-06T07:58:11.343727261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T07:58:24.665535441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:50400"
ts=2026-05-06T07:58:41.346459796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T07:59:03.669022063Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:59:11.348407504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T07:59:38.820468129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T07:59:41.350852456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T08:00:05.366761725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:48630"
ts=2026-05-06T08:00:11.353167961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T08:00:24.672207403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:44274"
level=debug ts=2026-05-06T08:00:41.002159754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:00:41.00389626Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:00:41.355811389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T08:00:54.674195531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:34360"
ts=2026-05-06T08:01:05.373076266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:43692"
ts=2026-05-06T08:01:11.358692775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T08:01:24.675850631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:48916"
level=debug ts=2026-05-06T08:01:27.063887487Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:01:37.319225909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:57408"
ts=2026-05-06T08:01:41.361449776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T08:01:54.677855766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:36044"
ts=2026-05-06T08:02:11.363883347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T08:02:17.386714473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:02:24.679756458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:43256"
ts=2026-05-06T08:02:35.382693217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:41136"
ts=2026-05-06T08:02:37.323223665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:44256"
ts=2026-05-06T08:02:41.366034732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T08:02:54.682002527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:35688"
level=debug ts=2026-05-06T08:02:56.71557329Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:03:11.36772845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T08:03:24.683481254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:58416"
ts=2026-05-06T08:03:35.38815474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:41346"
ts=2026-05-06T08:03:41.370525565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T08:03:58.377554076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:04:07.32987773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:52822"
ts=2026-05-06T08:04:11.372519568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T08:04:41.374331586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T08:04:43.366967314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:04:43.368975221Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:04:54.6919402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:47104"
ts=2026-05-06T08:05:11.376858357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T08:05:20.731547355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:05:35.400906704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:52946"
ts=2026-05-06T08:05:41.379105639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T08:05:54.696124055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:59422"
level=debug ts=2026-05-06T08:06:04.606417601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:06:07.342073881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:40934"
ts=2026-05-06T08:06:11.381376613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T08:06:24.69782768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:38792"
ts=2026-05-06T08:06:41.383614456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T08:06:43.670806659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:06:54.700049743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:38870"
ts=2026-05-06T08:07:05.410868924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:33892"
ts=2026-05-06T08:07:07.347481823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:45822"
ts=2026-05-06T08:07:11.385907539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=error ts=2026-05-06T08:07:20.386147468Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:07:20.388430675Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:07:20.390357949Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:07:20.390387245Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:07:41.388245089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T08:07:45.705133154Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:08:11.390083322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T08:08:41.39246458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T08:08:41.553027363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:08:41.55545328Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:09:05.423770476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:52178"
ts=2026-05-06T08:09:11.39532137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T08:09:21.120109534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:09:35.426997564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:34546"
ts=2026-05-06T08:09:41.397203412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T08:10:07.359204628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:38148"
ts=2026-05-06T08:10:11.398980922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T08:10:23.319752372Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:10:35.433738169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:55006"
ts=2026-05-06T08:10:37.361409285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:35238"
ts=2026-05-06T08:10:41.40098118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T08:11:11.40342795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T08:11:13.429957935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:11:35.440455385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:58154"
ts=2026-05-06T08:11:41.405187536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T08:11:50.361769657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:12:11.407628073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T08:12:35.287214169Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:12:35.288906213Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:12:41.409809451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T08:12:54.729016216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:36782"
ts=2026-05-06T08:13:11.411726885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T08:13:35.452980904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:47290"
level=debug ts=2026-05-06T08:13:35.505276301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:13:41.413175411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T08:14:05.455635686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:54504"
ts=2026-05-06T08:14:11.414759715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T08:14:20.804998498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:14:35.45868916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:42858"
ts=2026-05-06T08:14:41.416895936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T08:14:54.739846244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:37746"
ts=2026-05-06T08:15:05.462238813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:55476"
ts=2026-05-06T08:15:11.419022515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T08:15:15.618514114Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:15:24.742058093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:47318"
ts=2026-05-06T08:15:41.420867934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T08:16:04.557048828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:16:05.468788797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:34228"
ts=2026-05-06T08:16:11.422653712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T08:16:41.424687315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T08:16:45.804918289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:16:45.806692278Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:17:07.395030525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:47226"
ts=2026-05-06T08:17:11.426950844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=error ts=2026-05-06T08:17:20.386888057Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:17:20.389082905Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:17:20.391039372Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:17:20.391063907Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:17:24.749743133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:50670"
level=debug ts=2026-05-06T08:17:29.509249663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:17:41.429711215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T08:17:54.752197333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:38334"
ts=2026-05-06T08:18:11.432782423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T08:18:15.741948004Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:18:37.399698312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:57212"
ts=2026-05-06T08:18:41.435266849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T08:19:11.438174351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T08:19:12.794412269Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:19:24.75743737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:46766"
ts=2026-05-06T08:19:35.4905104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:50834"
ts=2026-05-06T08:19:41.440685359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T08:20:09.246232269Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:20:11.443454717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T08:20:41.445699177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T08:21:00.955572975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:21:00.957350643Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:21:11.447936577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T08:21:37.411466403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:37762"
ts=2026-05-06T08:21:41.450259265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T08:21:55.001282805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:22:11.452412419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T08:22:24.774025327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:33440"
ts=2026-05-06T08:22:35.508651422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:39162"
ts=2026-05-06T08:22:41.453873393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T08:22:44.260346749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:22:54.775937202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:34406"
ts=2026-05-06T08:23:05.511546646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:43924"
ts=2026-05-06T08:23:11.455751876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T08:23:35.822625765Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:23:41.457259625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T08:24:11.459537709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T08:24:31.144127222Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:24:41.461873985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T08:25:08.176588267Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:25:08.178140197Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:25:11.463466993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T08:25:24.787224924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:38110"
ts=2026-05-06T08:25:37.42552975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:45938"
ts=2026-05-06T08:25:41.465739637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T08:25:52.807329474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:26:11.467757467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T08:26:24.790845396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:36170"
ts=2026-05-06T08:26:35.532482929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:45588"
ts=2026-05-06T08:26:37.430950769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:59702"
ts=2026-05-06T08:26:41.470578184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T08:26:43.409779571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:27:05.535006635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:58936"
ts=2026-05-06T08:27:07.432987543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:55402"
ts=2026-05-06T08:27:11.473313814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=error ts=2026-05-06T08:27:20.389089643Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:27:20.391276514Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:27:20.393243241Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:27:20.393268415Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-06T08:27:35.168701411Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:27:37.434831376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:33628"
ts=2026-05-06T08:27:41.475593507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T08:27:54.797345922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:59546"
ts=2026-05-06T08:28:11.477286952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T08:28:32.144648903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:28:37.439162574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:59984"
ts=2026-05-06T08:28:41.479636105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T08:29:05.548470895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:42052"
ts=2026-05-06T08:29:11.482050363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T08:29:12.374613345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:29:12.376132804Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:29:24.803387101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:47712"
ts=2026-05-06T08:29:37.443391215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:42702"
ts=2026-05-06T08:29:41.48425153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T08:30:00.293014557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:30:11.486714929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T08:30:41.489806088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T08:30:58.584914269Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:31:05.560416119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:48986"
ts=2026-05-06T08:31:07.449035676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:55682"
ts=2026-05-06T08:31:11.492711607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T08:31:37.451267323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:39888"
ts=2026-05-06T08:31:41.495634077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T08:31:45.708630978Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:32:11.49768614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T08:32:35.569953305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:41296"
level=debug ts=2026-05-06T08:32:37.588035125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:32:41.499371363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T08:33:11.501770119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T08:33:17.684729922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:33:17.686385305Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:33:35.576052133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:40938"
ts=2026-05-06T08:33:41.504032035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T08:33:54.82157985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:48382"
ts=2026-05-06T08:34:11.506962981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T08:34:12.009837573Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:34:35.582788226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:52380"
ts=2026-05-06T08:34:37.463765079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:57722"
ts=2026-05-06T08:34:41.509851466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T08:34:54.824937781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:33356"
level=debug ts=2026-05-06T08:35:00.551396349Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:35:11.512584534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T08:35:24.826593678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:39984"
ts=2026-05-06T08:35:41.514945705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T08:35:47.280102718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:36:05.591592742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:51600"
ts=2026-05-06T08:36:11.517223651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
ts=2026-05-06T08:36:24.832288345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:51628"
ts=2026-05-06T08:36:37.470642617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:48230"
ts=2026-05-06T08:36:41.518918055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d26cefc9 10.129.0.85:7946"
level=debug ts=2026-05-06T08:36:42.528933537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:37:11.52107921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=error ts=2026-05-06T08:37:20.384115934Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:37:20.386168804Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:37:20.387992194Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T08:37:20.388015369Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-06T08:37:35.863207094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T08:37:35.86528166Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T08:37:41.523887354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T08:38:11.52707089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
level=debug ts=2026-05-06T08:38:24.530415548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:38:24.840404007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:33882"
ts=2026-05-06T08:38:41.529897549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-nvcst-81b13ff6 10.130.0.57:7946"
ts=2026-05-06T08:39:11.532916515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T08:39:24.845832204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:51998"
level=debug ts=2026-05-06T08:39:25.859643966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T08:39:41.535088006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
ts=2026-05-06T08:39:54.847461435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:50504"
ts=2026-05-06T08:40:05.617475571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.57:37272"
ts=2026-05-06T08:40:11.536898038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xv7bv-409ecf5d 10.129.0.80:7946"
level=debug ts=2026-05-06T08:40:16.487139846Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"