level=info ts=2026-04-16T15:24:10.649106437Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-16T15:24:10.649146447Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-16T15:24:10.650501706Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-16T15:24:10.650917955Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-16T15:24:10.65101149Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.82 inf=eth0
level=debug ts=2026-04-16T15:24:10.652325729Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-16T15:24:10.65240884Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.82 inf=eth0
level=info ts=2026-04-16T15:24:10.652517776Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-5d0a6552
ts=2026-04-16T15:24:10.652611977Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-16T15:24:10.652628497Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.82 advertisePort=7946
level=debug ts=2026-04-16T15:24:10.652649034Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.82 advertisePort=7946
level=info ts=2026-04-16T15:24:10.652798092Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-16T15:24:10.662733619Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-16T15:24:10.662763777Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-16T15:24:10.662779424Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=9.983414ms
level=error ts=2026-04-16T15:24:10.662796552Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-16T15:24:10.662811879Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-16T15:24:10.666065411Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-16T15:24:10.666084016Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-16T15:24:10.671848798Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-16T15:24:10.671881996Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-16T15:24:10.671892806Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-16T15:24:10.671932336Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-16T15:24:10.671948956Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-16T15:24:10.671980726Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-16T15:24:10.672107799Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-16T15:24:10.672129422Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-16T15:24:10.672281393Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-16T15:24:10.672294166Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-16T15:24:10.672327059Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-16T15:24:10.67234641Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-16T15:24:10.672391576Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-16T15:24:10.672624727Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-16T15:24:10.818399389Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-16T15:24:10.818465438Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-04-16T15:24:11.689200464Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-16T15:24:11.689229275Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-16T15:24:11.818817626Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-16T15:24:11.997424696Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-16T15:24:11.997469043Z caller=loki.go:599 msg="Loki started" startup_time=1.428735381s
ts=2026-04-16T15:24:13.905693721Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-16T15:24:13.905721287Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-16T15:24:16.997858273Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-16T15:24:16.997913855Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-16T15:24:18.524192196Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-16T15:24:18.524219275Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-16T15:24:25.232157611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T15:24:33.212432787Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.78:7946"
ts=2026-04-16T15:24:33.214565502Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.75:7946: dial tcp 10.128.0.75:7946: connect: connection refused"
ts=2026-04-16T15:24:33.215486329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.78:7946"
ts=2026-04-16T15:24:33.216954775Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.81:7946: dial tcp 10.128.0.81:7946: connect: connection refused"
ts=2026-04-16T15:24:33.217046286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.82:7946"
ts=2026-04-16T15:24:33.217079511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:55406"
ts=2026-04-16T15:24:33.218237052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.74:7946"
ts=2026-04-16T15:24:33.21955545Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.77:7946: dial tcp 10.128.0.77:7946: connect: connection refused"
level=info ts=2026-04-16T15:24:33.219567408Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=22.55674945s
ts=2026-04-16T15:24:34.502771369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:49644"
ts=2026-04-16T15:24:36.355983123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:43848"
level=debug ts=2026-04-16T15:24:42.128568619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T15:24:44.762114587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:35414"
ts=2026-04-16T15:24:54.001056939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
level=debug ts=2026-04-16T15:25:04.753996858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T15:25:24.002042859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
level=debug ts=2026-04-16T15:25:28.100486503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T15:25:54.003397797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:25:55.07913688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:57878"
level=debug ts=2026-04-16T15:25:59.619989714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-16T15:25:59.621530702Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T15:26:24.005777794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
level=debug ts=2026-04-16T15:26:47.277479068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T15:26:54.00775554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T15:27:16.176360043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:39544"
ts=2026-04-16T15:27:24.009380889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T15:27:33.852411733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:41586"
level=debug ts=2026-04-16T15:27:41.815644485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T15:27:54.010882998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T15:28:24.013007919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
level=debug ts=2026-04-16T15:28:41.890475089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T15:28:54.014424081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T15:29:03.856967734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:36166"
ts=2026-04-16T15:29:24.015967145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
level=debug ts=2026-04-16T15:29:48.089264491Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T15:29:54.018015038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T15:30:16.187220685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:59014"
ts=2026-04-16T15:30:24.019642094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:30:25.103041623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:56014"
level=debug ts=2026-04-16T15:30:46.716033305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-16T15:30:46.717606622Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T15:30:54.020861893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T15:31:03.863083073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:57540"
ts=2026-04-16T15:31:24.022983202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T15:31:33.864679474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:45132"
level=debug ts=2026-04-16T15:31:34.839624335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:31:54.024607831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
level=debug ts=2026-04-16T15:32:14.168717367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:32:24.025440751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T15:32:25.113575188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52616"
ts=2026-04-16T15:32:33.867906089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:60820"
ts=2026-04-16T15:32:54.02721389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:32:55.115383361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:40432"
level=debug ts=2026-04-16T15:32:55.737865061Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:33:16.199902454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:49362"
ts=2026-04-16T15:33:24.028711011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:33:25.117276307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:48378"
level=debug ts=2026-04-16T15:33:47.212259242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:33:54.030474454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
level=info ts=2026-04-16T15:34:16.997929258Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-16T15:34:17.001336169Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T15:34:17.004417908Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T15:34:17.006475723Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T15:34:17.006516734Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-16T15:34:17.006533663Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-16T15:34:24.031671015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T15:34:25.122054357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:43350"
level=debug ts=2026-04-16T15:34:25.25147199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T15:34:25.253342966Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T15:34:33.873340421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39584"
ts=2026-04-16T15:34:54.032873789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T15:35:03.874292064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:40968"
level=debug ts=2026-04-16T15:35:21.666229206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:35:24.035138415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T15:35:33.875163682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:46538"
ts=2026-04-16T15:35:54.03705169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:36:03.876893147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:55580"
level=debug ts=2026-04-16T15:36:19.195610791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:36:24.038473271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:36:25.131493686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:44198"
ts=2026-04-16T15:36:54.040952255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
level=debug ts=2026-04-16T15:37:00.889764639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:37:24.043337281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T15:37:25.136023252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:43406"
level=debug ts=2026-04-16T15:37:42.572979526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:37:54.045194671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:37:55.138213904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:43260"
ts=2026-04-16T15:38:03.882465783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:51446"
ts=2026-04-16T15:38:16.21869579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:58852"
level=debug ts=2026-04-16T15:38:23.601655209Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T15:38:23.604274865Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T15:38:24.04672025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T15:38:25.140746778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:50654"
ts=2026-04-16T15:38:33.883609155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39950"
ts=2026-04-16T15:38:54.048058769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T15:38:55.143442834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58488"
level=debug ts=2026-04-16T15:39:02.603393232Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:39:03.884823497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:36556"
ts=2026-04-16T15:39:16.223078409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:60932"
ts=2026-04-16T15:39:24.050386885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T15:39:46.22419324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:45594"
level=debug ts=2026-04-16T15:39:52.406940088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:39:54.052962525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T15:40:16.225339139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:35522"
ts=2026-04-16T15:40:24.055062679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T15:40:33.891083942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:42424"
level=debug ts=2026-04-16T15:40:41.910701471Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:40:46.227186299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:52238"
ts=2026-04-16T15:40:54.056637386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
level=debug ts=2026-04-16T15:41:16.720847273Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:41:24.057728249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:41:25.154828774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:46570"
ts=2026-04-16T15:41:33.893980931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:58420"
ts=2026-04-16T15:41:54.058901878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
level=debug ts=2026-04-16T15:41:54.643444088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T15:41:54.645054496Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T15:41:55.157299495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:34792"
ts=2026-04-16T15:42:03.895217143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:51346"
ts=2026-04-16T15:42:24.060107264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:42:25.159192315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:47046"
ts=2026-04-16T15:42:33.896974604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:35308"
level=debug ts=2026-04-16T15:42:35.72187465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:42:54.062008671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
level=debug ts=2026-04-16T15:43:12.470457819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:43:24.063580461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:43:25.164198496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:48688"
ts=2026-04-16T15:43:33.89869118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34340"
ts=2026-04-16T15:43:54.06483692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:43:55.167102298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:37372"
level=debug ts=2026-04-16T15:44:06.330725852Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T15:44:17.011514979Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T15:44:17.013391568Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T15:44:17.015152299Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T15:44:17.015178792Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T15:44:24.066824506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
level=debug ts=2026-04-16T15:44:40.572091483Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:44:54.068256498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:44:55.170721871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:54966"
level=debug ts=2026-04-16T15:45:22.892745214Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T15:45:22.894297938Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T15:45:24.069670583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:45:33.907646279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:42840"
ts=2026-04-16T15:45:54.071500329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
level=debug ts=2026-04-16T15:46:07.825716812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:46:16.244874465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:44188"
ts=2026-04-16T15:46:24.073489399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:46:25.177307391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:34902"
ts=2026-04-16T15:46:54.075699233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
level=debug ts=2026-04-16T15:47:03.528743106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:47:24.077787808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:47:54.078753649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T15:47:55.184828074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59414"
level=debug ts=2026-04-16T15:47:59.093570588Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:48:03.916747605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:46596"
ts=2026-04-16T15:48:24.080090928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
level=debug ts=2026-04-16T15:48:36.356178143Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:48:46.252612222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:41878"
ts=2026-04-16T15:48:54.081221669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:49:03.920317364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:53620"
level=debug ts=2026-04-16T15:49:14.031087244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T15:49:14.032579783Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T15:49:24.082482265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:49:33.921688046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34724"
level=debug ts=2026-04-16T15:49:53.394845483Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:49:54.08448886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T15:50:03.923409434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:38324"
ts=2026-04-16T15:50:24.085909435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
level=debug ts=2026-04-16T15:50:29.565964247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:50:54.087366959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T15:51:03.926198311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:56062"
level=debug ts=2026-04-16T15:51:19.891084598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:51:24.088180911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:51:46.262739907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:59044"
ts=2026-04-16T15:51:54.090624156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
level=debug ts=2026-04-16T15:52:10.060357605Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:52:16.264448641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:48868"
ts=2026-04-16T15:52:24.093457803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T15:52:25.211039338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:34930"
ts=2026-04-16T15:52:54.095616875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
level=debug ts=2026-04-16T15:53:01.458221086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T15:53:01.46005126Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T15:53:24.097177725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
level=debug ts=2026-04-16T15:53:35.776285364Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:53:54.099039109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:54:16.273325473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:50646"
level=error ts=2026-04-16T15:54:17.010774155Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T15:54:17.012564137Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T15:54:17.014376232Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T15:54:17.014398672Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T15:54:24.100493412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:54:25.221417699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41270"
level=debug ts=2026-04-16T15:54:32.936132138Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:54:46.27409518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:59014"
ts=2026-04-16T15:54:54.101766113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:54:55.224071612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41000"
ts=2026-04-16T15:55:24.103417987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
level=debug ts=2026-04-16T15:55:25.07883746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:55:25.226664605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:47130"
ts=2026-04-16T15:55:33.943880191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:54864"
ts=2026-04-16T15:55:54.106908544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T15:56:03.945434089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:41702"
level=debug ts=2026-04-16T15:56:04.952257165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:56:24.108877954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T15:56:33.947345568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:57974"
ts=2026-04-16T15:56:54.110815545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
level=debug ts=2026-04-16T15:57:04.09158471Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T15:57:04.093127873Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T15:57:24.112709347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
level=debug ts=2026-04-16T15:57:46.117899381Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:57:54.114118357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T15:57:55.237953259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39324"
ts=2026-04-16T15:58:24.115855109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
level=debug ts=2026-04-16T15:58:31.242309666Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:58:33.95451057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:46732"
ts=2026-04-16T15:58:54.117798442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T15:58:55.243633546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:55614"
ts=2026-04-16T15:59:16.289290842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:33130"
level=debug ts=2026-04-16T15:59:22.467914828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T15:59:24.119790024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T15:59:46.290481068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:54274"
ts=2026-04-16T15:59:54.121914693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
level=debug ts=2026-04-16T15:59:58.477946988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:00:03.9596133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39286"
ts=2026-04-16T16:00:24.123502203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
level=debug ts=2026-04-16T16:00:37.628774148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T16:00:37.630977715Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:00:54.124498383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T16:01:03.964094925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:38548"
ts=2026-04-16T16:01:16.296468966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:55980"
ts=2026-04-16T16:01:24.126104882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
level=debug ts=2026-04-16T16:01:32.259053419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:01:46.297446326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:59316"
ts=2026-04-16T16:01:54.127707364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T16:02:24.129898629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T16:02:25.259501404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:40022"
level=debug ts=2026-04-16T16:02:33.561570918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:02:46.299790682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:35878"
ts=2026-04-16T16:02:54.131479601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T16:03:16.300805718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:39656"
level=debug ts=2026-04-16T16:03:19.000970695Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:03:24.133496761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T16:03:33.972156814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:55940"
ts=2026-04-16T16:03:46.302303113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:34560"
ts=2026-04-16T16:03:54.13637694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T16:04:03.97350256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:38146"
level=debug ts=2026-04-16T16:04:10.073337333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T16:04:17.009836177Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:04:17.011539376Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:04:17.013157303Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:04:17.013183667Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:04:24.138403101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T16:04:25.268308896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:53276"
ts=2026-04-16T16:04:33.974798658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:40002"
ts=2026-04-16T16:04:54.140883604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T16:05:03.97636162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37030"
level=debug ts=2026-04-16T16:05:08.222378275Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T16:05:08.224139927Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:05:24.14303629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
level=debug ts=2026-04-16T16:05:44.673375144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:05:54.144232953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T16:05:55.276511444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:43686"
ts=2026-04-16T16:06:03.979559846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34692"
ts=2026-04-16T16:06:24.145254463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
level=debug ts=2026-04-16T16:06:25.043424343Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:06:46.314007297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:48596"
ts=2026-04-16T16:06:54.146513391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T16:06:55.281353457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:60804"
level=debug ts=2026-04-16T16:07:20.781699786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:07:24.147859828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T16:07:46.316640726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:52612"
ts=2026-04-16T16:07:54.148729999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
level=debug ts=2026-04-16T16:08:02.347908259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:08:16.317559836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:56696"
ts=2026-04-16T16:08:24.149496462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T16:08:25.288458898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:51796"
level=debug ts=2026-04-16T16:08:45.124928159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T16:08:45.126894852Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:08:54.150893545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
level=debug ts=2026-04-16T16:09:21.622366842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:09:24.152714074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T16:09:46.322187821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:49270"
ts=2026-04-16T16:09:54.154703374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
level=debug ts=2026-04-16T16:09:59.537552593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:10:24.156386275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T16:10:25.299037984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:47466"
level=debug ts=2026-04-16T16:10:43.870456903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:10:54.157639826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T16:11:03.998078024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:47338"
ts=2026-04-16T16:11:16.326818214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:46866"
ts=2026-04-16T16:11:24.159690046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
level=debug ts=2026-04-16T16:11:29.517748934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:11:46.327663685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:52918"
ts=2026-04-16T16:11:54.161395071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T16:11:55.306703835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49696"
ts=2026-04-16T16:12:04.001638667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:50376"
ts=2026-04-16T16:12:16.329170503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:45348"
level=debug ts=2026-04-16T16:12:17.525289786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T16:12:17.526875542Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:12:24.162496072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T16:12:54.163797441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T16:12:55.311744618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52784"
ts=2026-04-16T16:13:04.003831389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:35612"
level=debug ts=2026-04-16T16:13:08.299179423Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:13:24.165660465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T16:13:46.332927104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:41682"
ts=2026-04-16T16:13:54.167413884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
level=debug ts=2026-04-16T16:13:54.343566413Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:13:55.317356183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:33946"
ts=2026-04-16T16:14:16.334538403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:40570"
level=error ts=2026-04-16T16:14:17.011692718Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:14:17.013497078Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:14:17.015213271Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T16:14:17.015239699Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:14:24.168948962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
level=debug ts=2026-04-16T16:14:41.00100393Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:14:54.170396457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T16:15:04.010531286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:58862"
level=debug ts=2026-04-16T16:15:18.535110687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:15:24.171362004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T16:15:54.172872764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
level=debug ts=2026-04-16T16:16:03.787048312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T16:16:03.788555057Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T16:16:24.174473376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T16:16:34.014158774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34806"
ts=2026-04-16T16:16:54.176072804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
level=debug ts=2026-04-16T16:16:56.678528743Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:17:24.177326153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T16:17:25.335241667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:57968"
ts=2026-04-16T16:17:54.179698518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
level=debug ts=2026-04-16T16:17:55.082282486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:18:04.020329348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:50834"
ts=2026-04-16T16:18:24.181120466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"
ts=2026-04-16T16:18:25.340183335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:36414"
ts=2026-04-16T16:18:34.021518665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52336"
ts=2026-04-16T16:18:46.349573448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:35942"
ts=2026-04-16T16:18:54.183217694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
level=debug ts=2026-04-16T16:18:55.436032853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:19:04.022295648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:50306"
ts=2026-04-16T16:19:16.35145207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:50084"
ts=2026-04-16T16:19:24.185322495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-07fc12b0 10.129.0.78:7946"
ts=2026-04-16T16:19:25.345498707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:57304"
level=debug ts=2026-04-16T16:19:41.804347181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T16:19:54.187119052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-z87mm-6810299f 10.128.0.74:7946"
ts=2026-04-16T16:20:24.188434504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fd9ln-65dea685 10.128.0.78:7946"