level=info ts=2026-05-14T00:02:11.698930801Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-14T00:02:11.698967363Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-14T00:02:11.700045124Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-14T00:02:11.700318312Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-14T00:02:11.700384368Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.67 inf=eth0
level=debug ts=2026-05-14T00:02:11.701091177Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-14T00:02:11.701148014Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.67 inf=eth0
level=info ts=2026-05-14T00:02:11.701212747Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-e933de01
ts=2026-05-14T00:02:11.701294214Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-14T00:02:11.701305173Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.67 advertisePort=7946
level=debug ts=2026-05-14T00:02:11.701319444Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.67 advertisePort=7946
level=info ts=2026-05-14T00:02:11.701427258Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-14T00:02:11.708111237Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-14T00:02:11.708128269Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-14T00:02:11.708138564Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.713203ms
level=error ts=2026-05-14T00:02:11.7081472Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-14T00:02:11.708157763Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-14T00:02:11.710375859Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-14T00:02:11.710388412Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-14T00:02:11.715126538Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-14T00:02:11.715187447Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-14T00:02:11.715194121Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-14T00:02:11.715210669Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-14T00:02:11.71521737Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-14T00:02:11.715260502Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-14T00:02:11.715307913Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-14T00:02:11.715324366Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-14T00:02:11.715437586Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-14T00:02:11.715455882Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-14T00:02:11.715525416Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-14T00:02:11.715515132Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-14T00:02:11.715541332Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-14T00:02:11.715725718Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-14T00:02:11.715744358Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-14T00:02:11.715773573Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-14T00:02:12.716461424Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
ts=2026-05-14T00:02:12.819319327Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-14T00:02:12.819347161Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-14T00:02:12.841394925Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-14T00:02:12.841445614Z caller=loki.go:615 msg="Loki started" startup_time=1.199683401s
ts=2026-05-14T00:02:15.576407102Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-14T00:02:15.576430306Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-14T00:02:17.842230842Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-14T00:02:17.842301584Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-14T00:02:20.767038604Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-14T00:02:20.767060091Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-14T00:02:29.051711774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T00:02:35.809410889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.67:7946"
ts=2026-05-14T00:02:35.809521415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:35446"
level=info ts=2026-05-14T00:02:35.809766801Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=24.101604809s
ts=2026-05-14T00:02:36.091146426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:58832"
ts=2026-05-14T00:02:37.017930635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:40280"
ts=2026-05-14T00:02:40.694670738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:53604"
ts=2026-05-14T00:02:43.946527811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:02:47.631726431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:46520"
level=debug ts=2026-05-14T00:02:49.204438321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T00:02:51.310328396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:59462"
level=debug ts=2026-05-14T00:03:06.044153183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T00:03:13.949213847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:03:20.039429041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:45164"
ts=2026-05-14T00:03:21.313325224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:54678"
level=debug ts=2026-05-14T00:03:29.462697258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T00:03:43.9518026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:03:47.637148273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:36020"
level=debug ts=2026-05-14T00:03:55.237218009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-14T00:03:55.238777656Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:04:13.954850328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:04:17.639587229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:54372"
ts=2026-05-14T00:04:43.957837902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=debug ts=2026-05-14T00:04:46.381102254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T00:04:47.641497099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:33992"
ts=2026-05-14T00:05:13.960173049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:05:17.643707011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:38536"
ts=2026-05-14T00:05:21.324795046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:35868"
level=debug ts=2026-05-14T00:05:36.206016667Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T00:05:43.962948991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:05:50.053752085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:42732"
ts=2026-05-14T00:06:13.965739099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=debug ts=2026-05-14T00:06:40.496290571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T00:06:43.968098864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:06:51.333312423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:56418"
ts=2026-05-14T00:07:13.970311223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:07:43.972401668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:07:47.658677598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:35828"
level=debug ts=2026-05-14T00:07:55.640576035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T00:08:13.974821721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:08:20.062720672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:44154"
ts=2026-05-14T00:08:21.341324668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:35876"
ts=2026-05-14T00:08:43.976855838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=debug ts=2026-05-14T00:08:53.47385714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-14T00:08:53.47542523Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:09:13.979188098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
level=debug ts=2026-05-14T00:09:29.466495613Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:09:43.981579291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:10:13.984298572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:10:17.672501888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:34786"
level=debug ts=2026-05-14T00:10:23.473682861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:10:43.986765945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:10:47.67487093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:36574"
ts=2026-05-14T00:11:13.98904436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:11:17.677154022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:55028"
level=debug ts=2026-05-14T00:11:23.849176119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:11:43.991747362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:11:47.679340933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:57112"
level=debug ts=2026-05-14T00:12:07.512445816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:12:13.993979944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:12:17.681131779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:48984"
level=info ts=2026-05-14T00:12:17.842299581Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-14T00:12:17.846025735Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T00:12:17.847877049Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T00:12:17.849648213Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T00:12:17.849672787Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-14T00:12:17.849684877Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-14T00:12:43.996818041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:12:51.365652951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:35488"
level=debug ts=2026-05-14T00:13:04.980751196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T00:13:04.982601155Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:13:14.000009996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:13:17.686258003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:59276"
ts=2026-05-14T00:13:20.087794881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:46774"
ts=2026-05-14T00:13:21.36840496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:37056"
ts=2026-05-14T00:13:44.002918201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:13:47.68812624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:37738"
level=debug ts=2026-05-14T00:13:56.09488597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:14:14.005206007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:14:20.092306599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:58630"
ts=2026-05-14T00:14:44.012249835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=debug ts=2026-05-14T00:14:51.207706028Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:15:14.014731703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:15:20.099272333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:33020"
ts=2026-05-14T00:15:21.377467058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:60938"
level=debug ts=2026-05-14T00:15:35.784717976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:15:44.016868214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:16:14.019013497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:16:17.703700861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:40578"
level=debug ts=2026-05-14T00:16:35.342400388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:16:44.023836546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:17:14.025934478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:17:21.385822979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:46160"
level=debug ts=2026-05-14T00:17:27.622681804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T00:17:27.624073011Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:17:44.028453981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
level=debug ts=2026-05-14T00:18:02.140439749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:18:14.031402573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:18:17.715742158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:43584"
level=debug ts=2026-05-14T00:18:38.241405443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:18:44.035674751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:18:47.718128031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:40410"
ts=2026-05-14T00:19:14.041751376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:19:20.119024212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:55982"
ts=2026-05-14T00:19:21.394795677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:56318"
level=debug ts=2026-05-14T00:19:35.47678474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:19:44.045738888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:19:47.722995913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:56490"
ts=2026-05-14T00:20:14.048349635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:20:17.724541989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:39262"
ts=2026-05-14T00:20:20.129786671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:56956"
level=debug ts=2026-05-14T00:20:32.980005582Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:20:44.050542199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:20:47.727783991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:39658"
ts=2026-05-14T00:20:50.133769678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:45902"
ts=2026-05-14T00:21:14.053097402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
level=debug ts=2026-05-14T00:21:22.46146612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T00:21:22.463524443Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:21:44.056478405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:21:47.736104736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:46652"
ts=2026-05-14T00:21:50.140418357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:56632"
level=debug ts=2026-05-14T00:21:58.857109271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:22:14.058943744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:22:17.742774845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:47796"
level=error ts=2026-05-14T00:22:17.854431527Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T00:22:17.85695281Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T00:22:17.859259749Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T00:22:17.859292359Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:22:20.14406538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:49356"
ts=2026-05-14T00:22:21.41407343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:35990"
level=debug ts=2026-05-14T00:22:39.211891789Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:22:44.061897543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:22:47.744602478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:37342"
ts=2026-05-14T00:23:14.064172122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=debug ts=2026-05-14T00:23:26.14422505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:23:44.066566842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:24:14.068912906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:24:17.753363939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:57444"
level=debug ts=2026-05-14T00:24:20.810497097Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:24:44.071395272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:24:50.155501006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:54456"
ts=2026-05-14T00:25:14.073890481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
level=debug ts=2026-05-14T00:25:15.454606433Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T00:25:15.456171496Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:25:44.076961283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:25:50.161936026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:48724"
level=debug ts=2026-05-14T00:25:50.842895348Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:25:51.441156452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:34318"
ts=2026-05-14T00:26:14.079602799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:26:17.766053695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:47726"
level=debug ts=2026-05-14T00:26:36.735222916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:26:44.082367756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:26:47.768571478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:47522"
ts=2026-05-14T00:27:14.084696321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:27:17.771081027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:55684"
level=debug ts=2026-05-14T00:27:18.888048293Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:27:21.446584651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:51030"
ts=2026-05-14T00:27:44.087001502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=debug ts=2026-05-14T00:28:07.421566094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:28:14.089405368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:28:21.452166459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:39354"
ts=2026-05-14T00:28:44.09247607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:28:50.176690158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:53654"
level=debug ts=2026-05-14T00:29:06.548758232Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T00:29:06.550757345Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:29:14.09526138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:29:21.45755367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:54508"
ts=2026-05-14T00:29:44.098711026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:29:47.786990306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:36872"
level=debug ts=2026-05-14T00:29:48.766929153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:29:50.181337344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:43414"
ts=2026-05-14T00:30:14.101400463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:30:20.184452909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:56158"
level=debug ts=2026-05-14T00:30:36.360720495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:30:44.104338387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:30:51.464706777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:38788"
ts=2026-05-14T00:31:14.107420589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
level=debug ts=2026-05-14T00:31:14.579503449Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:31:44.110883375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:31:47.800501034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:36216"
ts=2026-05-14T00:31:50.190280945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:46724"
level=debug ts=2026-05-14T00:31:55.719744095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:32:14.113152758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=error ts=2026-05-14T00:32:17.854846849Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T00:32:17.856583441Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T00:32:17.858212583Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T00:32:17.858233616Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:32:21.474163824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:41140"
ts=2026-05-14T00:32:44.115666382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
level=debug ts=2026-05-14T00:32:52.964766812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T00:32:52.966347028Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:33:14.11827932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:33:20.19825008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:40544"
ts=2026-05-14T00:33:44.120312548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=debug ts=2026-05-14T00:33:45.526402994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:34:14.122569617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:34:17.815338712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:35512"
level=debug ts=2026-05-14T00:34:25.553439808Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:34:44.124970212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:34:47.817017306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:46756"
level=debug ts=2026-05-14T00:35:07.4319905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:35:14.127957361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:35:20.206662523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:35286"
ts=2026-05-14T00:35:44.131268567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:35:47.821926834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:53412"
level=debug ts=2026-05-14T00:35:52.284342111Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:36:14.134250503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
level=debug ts=2026-05-14T00:36:43.124020366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T00:36:43.125469128Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:36:44.136751894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:36:50.214247566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:37192"
ts=2026-05-14T00:37:14.138588876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:37:21.50056133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:48136"
level=debug ts=2026-05-14T00:37:27.898768329Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:37:44.14149926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:37:51.5039398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:53496"
ts=2026-05-14T00:38:14.144413464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:38:20.223742931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:38672"
level=debug ts=2026-05-14T00:38:26.665560941Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:38:44.146816563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:38:47.838828871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:35568"
level=debug ts=2026-05-14T00:39:02.547418751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:39:14.148708694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:39:21.512930598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:38258"
ts=2026-05-14T00:39:44.151122123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:39:47.84426909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:59296"
level=debug ts=2026-05-14T00:39:58.238739278Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:40:14.154415704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:40:21.51878309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:46322"
level=debug ts=2026-05-14T00:40:36.180335987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T00:40:36.182030029Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:40:44.157106212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:40:51.521392681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:47632"
ts=2026-05-14T00:41:14.160483303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:41:17.852746185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:37414"
level=debug ts=2026-05-14T00:41:18.549602551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:41:21.523882099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:35318"
ts=2026-05-14T00:41:44.163740989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
level=debug ts=2026-05-14T00:42:02.457339184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:42:14.167118381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
level=error ts=2026-05-14T00:42:17.85357911Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T00:42:17.855336463Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T00:42:17.857115879Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T00:42:17.857142837Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:42:17.857349355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:43826"
ts=2026-05-14T00:42:21.528405077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:35468"
level=debug ts=2026-05-14T00:42:40.13565069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:42:44.170097961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:43:14.172474278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=debug ts=2026-05-14T00:43:15.570152383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:43:17.86210346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:50346"
ts=2026-05-14T00:43:20.244760858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:38858"
ts=2026-05-14T00:43:44.175258813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:44:14.178514506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
level=debug ts=2026-05-14T00:44:16.579774027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T00:44:16.581413228Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:44:17.867589186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:39110"
ts=2026-05-14T00:44:21.539593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:51042"
ts=2026-05-14T00:44:44.181152806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:44:50.253677724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:39176"
ts=2026-05-14T00:44:51.542716175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:52280"
level=debug ts=2026-05-14T00:45:13.929413806Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:45:14.184037558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:45:44.187011902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:45:50.25911568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:59958"
ts=2026-05-14T00:45:51.547453636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:46352"
level=debug ts=2026-05-14T00:45:59.170671047Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:46:14.189751561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:46:17.87934294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:53050"
ts=2026-05-14T00:46:20.262339153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:59420"
ts=2026-05-14T00:46:44.192219876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:46:51.553300512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:44846"
level=debug ts=2026-05-14T00:46:56.912571169Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:47:14.195760219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:47:20.26726609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:59584"
level=debug ts=2026-05-14T00:47:31.977895184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:47:44.198765771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:47:50.269973252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:47310"
ts=2026-05-14T00:48:14.200871856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
level=debug ts=2026-05-14T00:48:16.257603839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T00:48:16.25910483Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:48:17.8918626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:45972"
ts=2026-05-14T00:48:20.272970655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:57076"
ts=2026-05-14T00:48:44.203058145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:48:50.275639934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:50826"
level=debug ts=2026-05-14T00:49:05.661741529Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:49:14.2051594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:49:44.208235653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:49:47.900509678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:60542"
level=debug ts=2026-05-14T00:49:49.280319555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:49:51.567691039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:57136"
ts=2026-05-14T00:50:14.211684838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:50:20.28310277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:52466"
level=debug ts=2026-05-14T00:50:37.848720101Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:50:44.214384447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:51:14.216939469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:51:17.908767705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:45838"
level=debug ts=2026-05-14T00:51:25.840192795Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:51:44.219203924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:51:47.910719685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:59082"
ts=2026-05-14T00:51:50.288875058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:35130"
ts=2026-05-14T00:52:14.221891592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=error ts=2026-05-14T00:52:17.855341339Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T00:52:17.857188762Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T00:52:17.858947675Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T00:52:17.858967878Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:52:21.57981315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:49576"
level=debug ts=2026-05-14T00:52:23.234600394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T00:52:23.236200145Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:52:44.22448135Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:52:50.294954199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:34586"
level=debug ts=2026-05-14T00:52:58.068541692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:53:14.226755151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:53:21.584598919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:42538"
level=debug ts=2026-05-14T00:53:34.90546488Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:53:44.229303128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:53:47.921568323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:45874"
ts=2026-05-14T00:54:14.231848016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:54:17.923517794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:45832"
ts=2026-05-14T00:54:21.590181774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:56110"
level=debug ts=2026-05-14T00:54:35.774293862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:54:44.2348736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:54:51.593087688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:51424"
ts=2026-05-14T00:55:14.238316354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:55:21.59618992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:35012"
level=debug ts=2026-05-14T00:55:28.536763564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:55:44.240988823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=debug ts=2026-05-14T00:56:06.27222687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T00:56:06.274054834Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T00:56:14.24390153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T00:56:17.934322579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:35688"
ts=2026-05-14T00:56:21.602803683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:44452"
ts=2026-05-14T00:56:44.247523584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
level=debug ts=2026-05-14T00:57:05.099812582Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:57:14.250032633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:57:17.939021432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:45562"
level=debug ts=2026-05-14T00:57:40.198460597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:57:44.252914617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T00:58:14.255956253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
level=debug ts=2026-05-14T00:58:29.34216088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:58:44.258352388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T00:58:50.325736302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:50272"
ts=2026-05-14T00:59:14.260411716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=debug ts=2026-05-14T00:59:23.778525772Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T00:59:44.263317955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
level=debug ts=2026-05-14T01:00:01.324663047Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T01:00:01.326256966Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T01:00:14.265846301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:00:17.956783683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:58510"
ts=2026-05-14T01:00:44.268612426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
level=debug ts=2026-05-14T01:00:49.297761266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:00:50.335890721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:42296"
ts=2026-05-14T01:01:14.271333233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T01:01:17.962109489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:34988"
ts=2026-05-14T01:01:20.338933411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:40584"
level=debug ts=2026-05-14T01:01:29.849365562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:01:44.274508133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T01:01:47.96409545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:56268"
ts=2026-05-14T01:02:14.277955636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
level=error ts=2026-05-14T01:02:17.851587708Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T01:02:17.853172289Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T01:02:17.85473111Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T01:02:17.854752126Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-14T01:02:19.448334959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:02:20.344176413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:35256"
ts=2026-05-14T01:02:21.63367357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:38264"
ts=2026-05-14T01:02:44.280671341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T01:02:50.3475529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:35354"
level=debug ts=2026-05-14T01:03:09.838816023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:03:14.283190763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T01:03:44.286165167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
level=debug ts=2026-05-14T01:03:45.824521559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T01:03:45.826081157Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T01:03:47.975377463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:44326"
ts=2026-05-14T01:04:14.288765373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
level=debug ts=2026-05-14T01:04:23.592542165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:04:44.291544665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:04:51.643408499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:35562"
ts=2026-05-14T01:05:14.294589129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
level=debug ts=2026-05-14T01:05:18.577981634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:05:20.359105206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:50576"
ts=2026-05-14T01:05:21.646434321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:38678"
ts=2026-05-14T01:05:44.297254093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:05:50.362418868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:39990"
level=debug ts=2026-05-14T01:06:03.886135022Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:06:14.29928505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:06:21.652208882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:39516"
ts=2026-05-14T01:06:44.30105052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=debug ts=2026-05-14T01:06:56.597111574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:07:14.303581222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T01:07:20.371816717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:36660"
ts=2026-05-14T01:07:21.655990705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:38212"
ts=2026-05-14T01:07:44.30616616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T01:07:47.999905172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:43234"
level=debug ts=2026-05-14T01:07:58.912816992Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T01:07:58.914373815Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T01:08:14.309277407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T01:08:21.660835304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:59156"
level=debug ts=2026-05-14T01:08:43.302583016Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:08:44.311873341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:08:51.664453828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:55644"
ts=2026-05-14T01:09:14.313935514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
level=debug ts=2026-05-14T01:09:19.750336704Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:09:44.316695488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T01:09:48.010409624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:41508"
ts=2026-05-14T01:09:50.384150382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:55084"
ts=2026-05-14T01:09:51.668865566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:36464"
ts=2026-05-14T01:10:14.31973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
level=debug ts=2026-05-14T01:10:17.648518977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:10:18.012097695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:51628"
ts=2026-05-14T01:10:21.672113077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:57576"
ts=2026-05-14T01:10:44.322340715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:10:48.014020502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:55012"
ts=2026-05-14T01:10:50.390291092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:57064"
ts=2026-05-14T01:11:14.324539684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=debug ts=2026-05-14T01:11:16.187440796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:11:18.015902787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:33586"
ts=2026-05-14T01:11:44.32712677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
level=debug ts=2026-05-14T01:12:00.863292814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T01:12:00.864823552Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T01:12:14.330079707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
level=error ts=2026-05-14T01:12:17.853645943Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T01:12:17.85553328Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T01:12:17.857384737Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T01:12:17.857409855Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T01:12:21.681078687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:55584"
ts=2026-05-14T01:12:44.332905417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T01:12:50.402053277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:51210"
level=debug ts=2026-05-14T01:12:59.713750952Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:13:14.336052555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T01:13:18.028156018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:42084"
ts=2026-05-14T01:13:20.404851624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:44104"
ts=2026-05-14T01:13:44.338573205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
level=debug ts=2026-05-14T01:13:46.256988955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:13:50.407791492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:32972"
ts=2026-05-14T01:14:14.34206568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
level=debug ts=2026-05-14T01:14:32.962004854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:14:44.34433895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=debug ts=2026-05-14T01:15:11.672910895Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:15:14.346527907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:15:18.041876214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:38194"
ts=2026-05-14T01:15:44.349817884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
level=debug ts=2026-05-14T01:16:12.517110968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T01:16:12.518886896Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T01:16:14.352827614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T01:16:21.704925469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:35550"
ts=2026-05-14T01:16:44.355729741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:16:48.049970013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:41964"
level=debug ts=2026-05-14T01:16:48.184195921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:17:14.357771176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:17:20.424108056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:45524"
ts=2026-05-14T01:17:21.711437358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:53816"
ts=2026-05-14T01:17:44.35958772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=debug ts=2026-05-14T01:17:47.032835965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:18:14.362154188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=debug ts=2026-05-14T01:18:38.385166149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:18:44.36418647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T01:18:50.432202423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:45432"
ts=2026-05-14T01:19:14.366702899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T01:19:18.063074304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:54630"
level=debug ts=2026-05-14T01:19:20.221669265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:19:20.435549632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:51470"
ts=2026-05-14T01:19:44.369521359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T01:19:48.065241421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:50948"
ts=2026-05-14T01:19:51.722589122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:41590"
ts=2026-05-14T01:20:14.372184719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T01:20:18.067524697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:46116"
level=debug ts=2026-05-14T01:20:18.627516023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T01:20:18.629063391Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T01:20:20.440180724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:57348"
ts=2026-05-14T01:20:44.375810665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T01:20:48.069730316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:59184"
ts=2026-05-14T01:21:14.378778009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
level=debug ts=2026-05-14T01:21:14.607499173Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:21:18.072282289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:56420"
ts=2026-05-14T01:21:20.446305071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:51298"
ts=2026-05-14T01:21:44.381777551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T01:21:48.074773838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:41480"
ts=2026-05-14T01:21:51.732981202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:36910"
level=debug ts=2026-05-14T01:22:01.316666335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:22:14.384342881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=error ts=2026-05-14T01:22:17.853512579Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T01:22:17.85523713Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T01:22:17.856992938Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T01:22:17.857017473Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T01:22:20.45212572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:50520"
ts=2026-05-14T01:22:21.736160967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:49348"
level=debug ts=2026-05-14T01:22:43.513368418Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:22:44.386542064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T01:23:14.389510763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T01:23:18.082699584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:39398"
level=debug ts=2026-05-14T01:23:25.660777316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:23:44.392800213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T01:23:51.744134284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:50504"
ts=2026-05-14T01:24:14.395132703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:24:18.087525982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:39698"
ts=2026-05-14T01:24:21.746691795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:49094"
level=debug ts=2026-05-14T01:24:24.15943274Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T01:24:24.161067523Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T01:24:44.397400116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:25:14.402848486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T01:25:20.467264003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:47514"
ts=2026-05-14T01:25:21.750661769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:59876"
level=debug ts=2026-05-14T01:25:25.738556185Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:25:44.405804565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T01:25:48.095375453Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:53824"
ts=2026-05-14T01:26:14.409199559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T01:26:18.097139509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:57466"
ts=2026-05-14T01:26:21.755997955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:48498"
level=debug ts=2026-05-14T01:26:22.535770481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:26:44.411409504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:26:48.099878877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:44128"
ts=2026-05-14T01:26:50.473969534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:52800"
level=debug ts=2026-05-14T01:27:01.84103446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:27:14.413984039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:27:44.416267431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
level=debug ts=2026-05-14T01:27:47.218898749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:27:48.105291403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:45802"
ts=2026-05-14T01:28:14.418993884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T01:28:18.10707894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:60238"
level=debug ts=2026-05-14T01:28:38.573412755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T01:28:38.574880984Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T01:28:44.42184517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T01:28:50.483478864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:41384"
ts=2026-05-14T01:29:14.424760237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:29:21.768936464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:58468"
level=debug ts=2026-05-14T01:29:40.379756306Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:29:44.427443043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T01:30:14.430855803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
level=debug ts=2026-05-14T01:30:25.320085689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:30:44.433522485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:30:50.490257243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:54742"
ts=2026-05-14T01:30:51.775522822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:49498"
level=debug ts=2026-05-14T01:31:01.77532558Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:31:14.435961148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:31:21.778272138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:51990"
level=debug ts=2026-05-14T01:31:44.221220968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:31:44.438869558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T01:31:50.496266992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:33244"
ts=2026-05-14T01:32:14.441399836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
level=error ts=2026-05-14T01:32:17.854755719Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T01:32:17.856692494Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T01:32:17.85842901Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T01:32:17.85845192Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T01:32:21.782369313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:58410"
level=debug ts=2026-05-14T01:32:39.81274181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T01:32:39.814258713Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T01:32:44.44426916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:32:48.133853374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:59500"
ts=2026-05-14T01:32:50.501763814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:46106"
ts=2026-05-14T01:33:14.446822443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
level=debug ts=2026-05-14T01:33:26.540441611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:33:44.449416027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:33:48.13942773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:48216"
level=debug ts=2026-05-14T01:34:06.130458845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:34:14.451687052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:34:18.141487127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:46330"
ts=2026-05-14T01:34:44.454150462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T01:34:51.794378212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:40828"
level=debug ts=2026-05-14T01:35:01.909367265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:35:14.456347877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:35:21.797555676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:33820"
ts=2026-05-14T01:35:44.457874545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
level=debug ts=2026-05-14T01:35:47.997998653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:35:51.80031527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:32970"
ts=2026-05-14T01:36:14.459946342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
level=debug ts=2026-05-14T01:36:25.477674059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T01:36:25.479210267Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T01:36:44.463178719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T01:36:51.805369837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:49416"
level=debug ts=2026-05-14T01:37:00.827589135Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:37:14.466264174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T01:37:18.158518558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:52232"
ts=2026-05-14T01:37:44.468853824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
ts=2026-05-14T01:37:48.160884064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:57012"
ts=2026-05-14T01:37:50.525032977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:34708"
level=debug ts=2026-05-14T01:37:53.593854714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:38:14.47131215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8n42v-88d91f01 10.130.0.91:7946"
ts=2026-05-14T01:38:44.473936698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
level=debug ts=2026-05-14T01:38:46.837684745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:38:50.52953791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:44036"
ts=2026-05-14T01:38:51.818110085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:34944"
ts=2026-05-14T01:39:14.476883149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:39:20.532224128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:34858"
level=debug ts=2026-05-14T01:39:22.189290316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T01:39:44.479429597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f5dd0430 10.129.0.80:7946"
ts=2026-05-14T01:39:48.173226076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:38622"
ts=2026-05-14T01:39:51.82134823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:39476"
ts=2026-05-14T01:40:14.4822139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mz4rw-74437e5a 10.130.0.89:7946"
level=debug ts=2026-05-14T01:40:18.174777957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T01:40:18.176247678Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T01:40:18.176271152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:47910"