level=info ts=2026-05-19T11:13:30.375329649Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-19T11:13:30.375354992Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-19T11:13:30.376428234Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-19T11:13:30.376734036Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-19T11:13:30.376808849Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.81 inf=eth0
level=debug ts=2026-05-19T11:13:30.377586251Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-19T11:13:30.377644666Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.81 inf=eth0
level=info ts=2026-05-19T11:13:30.377719463Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-63d5b07a
ts=2026-05-19T11:13:30.377820221Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-19T11:13:30.377832533Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.81 advertisePort=7946
level=debug ts=2026-05-19T11:13:30.377851662Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.81 advertisePort=7946
level=info ts=2026-05-19T11:13:30.377975746Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-19T11:13:30.384711266Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-19T11:13:30.384728477Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-19T11:13:30.38473896Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.765728ms
level=error ts=2026-05-19T11:13:30.384747756Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-19T11:13:30.384758777Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-19T11:13:30.386915827Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-19T11:13:30.386930696Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-19T11:13:30.391343406Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-19T11:13:30.391398491Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-19T11:13:30.391425429Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-19T11:13:30.391419304Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-19T11:13:30.39143512Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-19T11:13:30.391524832Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-19T11:13:30.391534919Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-05-19T11:13:30.391547301Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-19T11:13:30.391552869Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-05-19T11:13:30.391649976Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-19T11:13:30.391697755Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-19T11:13:30.391684586Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-19T11:13:30.391713476Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-19T11:13:30.391861972Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-19T11:13:30.391875042Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-19T11:13:30.391890099Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-19T11:13:31.393015764Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-19T11:13:31.552520559Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-19T11:13:31.552576119Z caller=loki.go:615 msg="Loki started" startup_time=1.23831274s
ts=2026-05-19T11:13:32.334593713Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-19T11:13:32.33461837Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T11:13:36.202084196Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-19T11:13:36.202107456Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-19T11:13:36.553012811Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-19T11:13:36.553064647Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-19T11:13:43.824621674Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-19T11:13:43.824643666Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-19T11:13:46.5132779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T11:13:48.523853136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52006"
ts=2026-05-19T11:13:51.807234603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48144"
ts=2026-05-19T11:13:51.943619545Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.65:7946: dial tcp 10.130.0.65:7946: connect: connection refused"
ts=2026-05-19T11:13:51.94372916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.81:7946"
ts=2026-05-19T11:13:51.94377383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:37078"
ts=2026-05-19T11:13:51.945806138Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.62:7946: dial tcp 10.130.0.62:7946: connect: connection refused"
ts=2026-05-19T11:13:51.946918641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.67:7946"
ts=2026-05-19T11:13:51.949913459Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.60:7946: dial tcp 10.130.0.60:7946: connect: connection refused"
ts=2026-05-19T11:13:51.95110209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.63:7946"
level=info ts=2026-05-19T11:13:51.952585658Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=21.567822417s
ts=2026-05-19T11:13:53.473472391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:54130"
ts=2026-05-19T11:13:59.732068608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:34976"
level=debug ts=2026-05-19T11:14:06.240769859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T11:14:06.34719796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T11:14:24.847584555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:54918"
level=debug ts=2026-05-19T11:14:28.222885964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T11:14:36.350456202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T11:14:48.480299147Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T11:15:06.352779166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T11:15:13.418944952Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-19T11:15:13.420658112Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T11:15:22.182873384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:51916"
ts=2026-05-19T11:15:24.851922723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:44390"
ts=2026-05-19T11:15:36.355924925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T11:15:42.523988915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T11:15:52.185963941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:54174"
ts=2026-05-19T11:15:59.742050068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:51460"
ts=2026-05-19T11:16:06.358582618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:16:36.361851292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T11:16:43.037045782Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T11:16:54.859489181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:39314"
ts=2026-05-19T11:17:06.364141482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:17:24.862111074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:34130"
ts=2026-05-19T11:17:36.367529918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T11:17:40.955021638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T11:17:54.865228837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:52566"
ts=2026-05-19T11:18:06.370617662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:18:22.195504879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49298"
ts=2026-05-19T11:18:36.373632441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T11:18:50.26638248Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T11:19:06.376671937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:19:24.87172491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:52524"
ts=2026-05-19T11:19:36.380255936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T11:19:39.982927492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-19T11:19:39.984534899Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T11:19:52.201985799Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:40724"
ts=2026-05-19T11:19:59.757207304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60344"
ts=2026-05-19T11:20:06.3831881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T11:20:30.097114204Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:20:36.386455084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:20:54.878285128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:40122"
ts=2026-05-19T11:21:06.389662056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T11:21:18.90300409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:21:36.39270643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T11:21:53.308570898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:21:54.883008391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:54624"
ts=2026-05-19T11:22:06.395180573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:22:36.398526837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T11:22:49.603055933Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:22:54.888193136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:41204"
ts=2026-05-19T11:22:59.769944765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45306"
ts=2026-05-19T11:23:06.401330835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:23:29.772176346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42330"
level=debug ts=2026-05-19T11:23:35.626042027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T11:23:35.628556969Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T11:23:36.404026873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=info ts=2026-05-19T11:23:36.553868642Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-19T11:23:36.556379637Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T11:23:36.558289706Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T11:23:36.560192316Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T11:23:36.560219805Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-19T11:23:36.560234516Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-19T11:23:59.774744682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47338"
ts=2026-05-19T11:24:06.407245238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T11:24:17.024499898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:24:22.21921302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:52806"
ts=2026-05-19T11:24:24.894106626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:53688"
ts=2026-05-19T11:24:29.777292521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:51170"
ts=2026-05-19T11:24:36.411128655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:24:59.78024084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41622"
ts=2026-05-19T11:25:06.41469356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T11:25:14.845565902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:25:22.224531226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:33252"
ts=2026-05-19T11:25:29.782997431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50880"
ts=2026-05-19T11:25:36.417948971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T11:25:54.900334282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:44908"
level=debug ts=2026-05-19T11:26:01.245805891Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:26:06.421270023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T11:26:24.903984005Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:48924"
ts=2026-05-19T11:26:29.787078231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39970"
ts=2026-05-19T11:26:36.424972248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T11:26:43.221364653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:27:06.428651376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T11:27:36.432001377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T11:27:39.084500891Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T11:27:39.086364298Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T11:28:06.436174373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T11:28:20.460560175Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:28:24.912645147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:39092"
ts=2026-05-19T11:28:29.796767679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60598"
ts=2026-05-19T11:28:36.439138291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:28:59.80014537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43938"
ts=2026-05-19T11:29:06.442881614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T11:29:13.953738665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:29:22.241421633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:45154"
ts=2026-05-19T11:29:24.917320025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:57882"
ts=2026-05-19T11:29:36.446682676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:29:59.805164755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:34360"
level=debug ts=2026-05-19T11:30:00.36406607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:30:06.449668211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:30:22.246603063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60192"
ts=2026-05-19T11:30:36.452608119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:30:52.250169366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:50946"
level=debug ts=2026-05-19T11:30:53.149656469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:30:59.810464534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37668"
ts=2026-05-19T11:31:06.456361285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T11:31:24.926115714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:47750"
ts=2026-05-19T11:31:29.813346146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39672"
level=debug ts=2026-05-19T11:31:35.707069953Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T11:31:35.708945382Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T11:31:36.459983468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T11:32:06.46371717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T11:32:13.875582033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:32:24.931356888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:49492"
ts=2026-05-19T11:32:36.467142601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T11:32:57.906686516Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:33:06.470194962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T11:33:29.822989282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50258"
ts=2026-05-19T11:33:36.473759824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=error ts=2026-05-19T11:33:36.565819006Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T11:33:36.567754846Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T11:33:36.569667225Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T11:33:36.569688103Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T11:33:47.632236571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:33:52.262780836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:37456"
ts=2026-05-19T11:33:59.826475388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47486"
ts=2026-05-19T11:34:06.477044378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T11:34:29.829651396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41074"
ts=2026-05-19T11:34:36.480006291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T11:34:49.762998757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:34:52.267419395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:33076"
ts=2026-05-19T11:34:54.941791174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:41526"
ts=2026-05-19T11:35:06.483325124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:35:29.834240246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41828"
ts=2026-05-19T11:35:36.48666681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T11:35:37.141946951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T11:35:37.145116918Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T11:35:59.837376677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47120"
ts=2026-05-19T11:36:06.488912856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:36:22.273783942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60188"
ts=2026-05-19T11:36:24.948384645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:49906"
level=debug ts=2026-05-19T11:36:29.938438903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:36:36.491865963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:36:52.277292376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:35630"
ts=2026-05-19T11:36:54.950906541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:53102"
ts=2026-05-19T11:36:59.842219881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43846"
ts=2026-05-19T11:37:06.494325917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T11:37:22.781616585Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:37:36.497035367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:37:59.847373876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53386"
ts=2026-05-19T11:38:06.499902939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T11:38:12.373092879Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:38:24.955834551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:46962"
ts=2026-05-19T11:38:36.50292284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:38:52.285124327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:44080"
level=debug ts=2026-05-19T11:38:52.742713936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:39:06.505680671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T11:39:29.855105028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33752"
ts=2026-05-19T11:39:36.509515738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T11:39:47.181088202Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T11:39:47.182648472Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T11:40:06.512976259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:40:22.291366407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:55240"
ts=2026-05-19T11:40:24.963766453Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:34996"
level=debug ts=2026-05-19T11:40:29.024563958Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:40:29.859782921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:35444"
ts=2026-05-19T11:40:36.516513081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T11:40:52.294959304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:46972"
ts=2026-05-19T11:40:54.967119036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:42230"
ts=2026-05-19T11:40:59.863194692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45330"
level=debug ts=2026-05-19T11:41:03.772128347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:41:06.519918029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T11:41:24.969856296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:60274"
ts=2026-05-19T11:41:36.522866277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:41:52.300043045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48364"
level=debug ts=2026-05-19T11:42:03.769506607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:42:06.525451155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:42:24.974876583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:40452"
ts=2026-05-19T11:42:29.870180936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53000"
ts=2026-05-19T11:42:36.52836966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T11:42:59.555547699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:43:06.530600807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:43:22.307427871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:42362"
ts=2026-05-19T11:43:36.533515444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=error ts=2026-05-19T11:43:36.566499042Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T11:43:36.568484996Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T11:43:36.570312761Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T11:43:36.570338203Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T11:43:44.163877561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T11:43:44.165603978Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T11:44:06.536693774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:44:24.982394032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:59462"
ts=2026-05-19T11:44:36.539617513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T11:44:37.424682175Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:44:59.88010922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33462"
ts=2026-05-19T11:45:06.542883162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:45:29.882550597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48316"
level=debug ts=2026-05-19T11:45:34.928805651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:45:36.545699917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:45:54.988645727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:52014"
ts=2026-05-19T11:46:06.548566691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T11:46:17.953097936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:46:29.887216975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59792"
ts=2026-05-19T11:46:36.552247924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:46:52.319783762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:58836"
level=debug ts=2026-05-19T11:46:54.494886042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:46:54.993194824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:58722"
ts=2026-05-19T11:46:59.890034545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53526"
ts=2026-05-19T11:47:06.556045304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T11:47:29.337445321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T11:47:29.339325973Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T11:47:36.559315156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:47:59.895341775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:36074"
ts=2026-05-19T11:48:06.562272069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T11:48:18.768863096Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:48:25.000292588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:45780"
ts=2026-05-19T11:48:36.564713379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:48:55.00272241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:50588"
ts=2026-05-19T11:49:06.566999218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T11:49:17.438203656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:49:25.005459428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:56390"
ts=2026-05-19T11:49:36.569377179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T11:49:57.723907365Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:49:59.903565393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45330"
ts=2026-05-19T11:50:06.572154934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:50:36.574903335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T11:50:52.767433922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:50:59.908253466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54218"
ts=2026-05-19T11:51:06.577975993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:51:36.581712761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T11:51:52.339662885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:52560"
level=debug ts=2026-05-19T11:51:54.748426347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T11:51:54.750504945Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T11:51:59.913547839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37802"
ts=2026-05-19T11:52:06.58534452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:52:36.587774688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T11:52:40.018813152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:52:52.344288446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:34688"
ts=2026-05-19T11:52:55.019111757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:49424"
ts=2026-05-19T11:53:06.59078574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:53:22.347706398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49664"
level=debug ts=2026-05-19T11:53:29.093992183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:53:29.920479174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39734"
level=error ts=2026-05-19T11:53:36.562895568Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T11:53:36.564520485Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T11:53:36.566832764Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T11:53:36.566853648Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T11:53:36.594378306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T11:54:05.747831937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:54:06.597599636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:54:36.600366931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T11:54:46.203155332Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:54:52.354551008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:34890"
ts=2026-05-19T11:55:06.603787354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T11:55:29.92958323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:36894"
level=debug ts=2026-05-19T11:55:32.982759894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T11:55:32.984442103Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T11:55:36.606428948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:56:06.609397833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T11:56:13.598249166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:56:22.361555309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:56796"
ts=2026-05-19T11:56:36.611721028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:56:52.365101075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:39140"
ts=2026-05-19T11:56:55.034166794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:41044"
level=debug ts=2026-05-19T11:57:04.435487617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:57:06.615066152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T11:57:22.368472367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:35704"
ts=2026-05-19T11:57:36.61806432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T11:57:43.774675597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:57:55.039312242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:58228"
ts=2026-05-19T11:58:06.620734789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T11:58:35.751837048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T11:58:36.623654241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T11:58:59.945952856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45880"
ts=2026-05-19T11:59:06.627148065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T11:59:25.047151801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:44660"
level=debug ts=2026-05-19T11:59:35.083546579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T11:59:35.085317895Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T11:59:36.63064885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T11:59:52.378207666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48074"
ts=2026-05-19T11:59:55.049732539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:43610"
ts=2026-05-19T12:00:06.633514811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:00:22.381567337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49794"
ts=2026-05-19T12:00:25.052822515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:44042"
ts=2026-05-19T12:00:29.952646273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48248"
level=debug ts=2026-05-19T12:00:31.212877351Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:00:36.636567057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T12:00:59.955151615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:44878"
ts=2026-05-19T12:01:06.639463884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T12:01:13.831114363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:01:22.387159831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:41718"
ts=2026-05-19T12:01:36.64285373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T12:01:59.959301731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39690"
ts=2026-05-19T12:02:06.646356555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T12:02:14.440990174Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:02:36.649131896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:02:52.393870684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:59832"
ts=2026-05-19T12:02:55.061691769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:45618"
level=debug ts=2026-05-19T12:02:58.460212596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:03:06.651936888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T12:03:22.397024446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:45746"
level=error ts=2026-05-19T12:03:36.566748202Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T12:03:36.568473958Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T12:03:36.570170991Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T12:03:36.570203649Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T12:03:36.65553069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T12:03:51.224682444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T12:03:51.226290834Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T12:04:06.658469256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:04:36.66101196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T12:04:50.288593023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:05:06.663968643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:05:22.408389939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:47784"
ts=2026-05-19T12:05:25.072293471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:45968"
ts=2026-05-19T12:05:36.667037849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T12:05:45.155832509Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:05:52.411202814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:38672"
ts=2026-05-19T12:06:06.670110768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:06:29.979267649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:51118"
ts=2026-05-19T12:06:36.67224254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T12:06:41.599742715Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:07:06.675108025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T12:07:18.065327984Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:07:22.418578123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:51104"
ts=2026-05-19T12:07:25.080033644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:36726"
ts=2026-05-19T12:07:36.678249688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T12:07:59.984787905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52526"
level=debug ts=2026-05-19T12:08:03.356880387Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T12:08:03.358453185Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T12:08:06.681142565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:08:22.423570937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:55788"
ts=2026-05-19T12:08:25.084902193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:56448"
ts=2026-05-19T12:08:36.6837699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T12:08:48.573378533Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:08:59.989345404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:36164"
ts=2026-05-19T12:09:06.686915178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T12:09:25.090185292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:43624"
ts=2026-05-19T12:09:29.992380156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:44072"
ts=2026-05-19T12:09:36.690969003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T12:09:46.102999132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:10:06.694060275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T12:10:33.134572074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:10:36.697448735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T12:10:59.998447028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59804"
ts=2026-05-19T12:11:06.701052153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T12:11:08.612510769Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:11:22.435887273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:38104"
ts=2026-05-19T12:11:25.098278096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:36474"
ts=2026-05-19T12:11:36.703826309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:11:58.616721598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T12:11:58.61909067Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T12:12:00.002315411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60376"
ts=2026-05-19T12:12:06.70714672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T12:12:22.440067614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60052"
ts=2026-05-19T12:12:25.103039626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:60612"
ts=2026-05-19T12:12:30.005016374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37858"
ts=2026-05-19T12:12:36.709791356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T12:12:40.565846067Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:12:52.443469951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:34960"
ts=2026-05-19T12:13:00.007744242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:40408"
ts=2026-05-19T12:13:06.712883729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T12:13:19.877867834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:13:25.107721039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:58744"
ts=2026-05-19T12:13:30.011485982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57572"
level=error ts=2026-05-19T12:13:36.564109292Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T12:13:36.566049345Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T12:13:36.567732575Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T12:13:36.567753795Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T12:13:36.715567341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T12:14:06.719112379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:14:14.627064578Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:14:36.722205603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:14:52.451880842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:58800"
ts=2026-05-19T12:15:06.725653047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:15:10.216339486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:15:36.728552383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T12:15:55.11677898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:44548"
ts=2026-05-19T12:16:06.731598572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T12:16:10.062570165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T12:16:10.064121462Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T12:16:22.45758229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:42672"
ts=2026-05-19T12:16:30.024720957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53558"
ts=2026-05-19T12:16:36.73482063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:16:52.408775187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:16:52.460912937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:36734"
ts=2026-05-19T12:16:55.120894734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:53530"
ts=2026-05-19T12:17:06.738325342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:17:22.463829801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:51736"
ts=2026-05-19T12:17:25.124017894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:56352"
ts=2026-05-19T12:17:36.74120174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T12:17:41.460508505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:17:55.126989505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:55752"
ts=2026-05-19T12:18:06.743908267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T12:18:24.889578823Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:18:25.130590818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:51668"
ts=2026-05-19T12:18:30.033193566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54534"
ts=2026-05-19T12:18:36.747635536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T12:18:52.468999922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48620"
ts=2026-05-19T12:19:06.750914908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T12:19:26.934257528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:19:30.038462806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:58824"
ts=2026-05-19T12:19:36.75358854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T12:20:06.756500075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T12:20:21.346187908Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T12:20:21.347698404Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T12:20:36.759665445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:20:52.477700234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:33526"
level=debug ts=2026-05-19T12:21:05.154330048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:21:06.762700024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T12:21:22.48109353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:56522"
ts=2026-05-19T12:21:36.765882782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:21:55.144200638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:47222"
level=debug ts=2026-05-19T12:22:05.174552924Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:22:06.769099451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T12:22:22.486839381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49170"
ts=2026-05-19T12:22:36.772401077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:22:49.892586965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:22:52.490551168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:32896"
ts=2026-05-19T12:22:55.148118182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:55246"
ts=2026-05-19T12:23:06.775549888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T12:23:25.634927825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:23:30.054541908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:58918"
level=error ts=2026-05-19T12:23:36.564960695Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T12:23:36.566912273Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T12:23:36.568485036Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T12:23:36.568508922Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T12:23:36.778564103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T12:23:55.152860252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:40494"
ts=2026-05-19T12:24:00.056894563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:51744"
ts=2026-05-19T12:24:06.781791087Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:24:19.570264854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T12:24:19.571988744Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T12:24:25.155689837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:48360"
ts=2026-05-19T12:24:36.784420354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:25:06.786929188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T12:25:18.316666322Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:25:22.501180987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:33310"
ts=2026-05-19T12:25:25.160485531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:55758"
ts=2026-05-19T12:25:36.790184789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:26:02.223969547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:26:06.793448048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:26:30.067471875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:35880"
ts=2026-05-19T12:26:36.796294121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T12:26:53.335978418Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:27:06.798977529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:27:36.801794995Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T12:27:44.3171237Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:27:52.511797602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:46906"
ts=2026-05-19T12:28:00.07390934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43562"
ts=2026-05-19T12:28:06.804866819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T12:28:36.808519692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T12:28:39.736691826Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T12:28:39.738248535Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T12:28:52.517601581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:51550"
ts=2026-05-19T12:29:00.077482441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60948"
ts=2026-05-19T12:29:06.811742375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T12:29:22.520590705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:39374"
level=debug ts=2026-05-19T12:29:28.628288076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:29:30.079600616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:51526"
ts=2026-05-19T12:29:36.815503834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T12:30:06.818628173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:30:17.257560516Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:30:36.822214861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:31:00.086965836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:34542"
ts=2026-05-19T12:31:06.824812061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:31:09.004004706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:31:22.529979833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:49224"
ts=2026-05-19T12:31:36.827853888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T12:31:59.086309973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:32:00.091359274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39204"
ts=2026-05-19T12:32:06.830574969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:32:25.183712045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:38622"
ts=2026-05-19T12:32:30.094093452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47796"
ts=2026-05-19T12:32:36.83338808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T12:32:49.94573191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T12:32:49.947263942Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T12:33:06.836582838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:33:25.188680726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:50024"
level=error ts=2026-05-19T12:33:36.56611727Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T12:33:36.567998612Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T12:33:36.569590536Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T12:33:36.569615009Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T12:33:36.839910278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T12:33:41.999068429Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:33:55.191452221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:34598"
ts=2026-05-19T12:34:06.843259788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:34:21.402812162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:34:36.846988765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:35:05.193078698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:35:06.850122753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T12:35:30.107806972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47002"
ts=2026-05-19T12:35:36.853752129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T12:35:52.546487472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:55102"
ts=2026-05-19T12:35:55.19979395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:46336"
level=debug ts=2026-05-19T12:36:00.058960653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:36:06.85670196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T12:36:22.549375135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:45770"
ts=2026-05-19T12:36:30.112627499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:51030"
ts=2026-05-19T12:36:36.860045719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:36:50.557533129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T12:36:50.559017923Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T12:37:06.863773774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T12:37:22.554295862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:32814"
ts=2026-05-19T12:37:25.206419985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:54292"
ts=2026-05-19T12:37:30.117176785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:36512"
ts=2026-05-19T12:37:36.86665312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
level=debug ts=2026-05-19T12:37:46.628890616Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:37:52.557539221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:40340"
ts=2026-05-19T12:38:00.120062368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45106"
ts=2026-05-19T12:38:06.869520386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:38:25.210258076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:46136"
level=debug ts=2026-05-19T12:38:30.061488929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:38:30.122782374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33548"
ts=2026-05-19T12:38:36.872669297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:38:55.21369343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:34458"
ts=2026-05-19T12:39:00.125109828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48748"
ts=2026-05-19T12:39:06.875727521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T12:39:15.30775226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:39:22.563589578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:47822"
ts=2026-05-19T12:39:36.87844926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:39:55.218080325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:51796"
level=debug ts=2026-05-19T12:40:05.271591126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:40:06.880884997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T12:40:36.883637275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:40:40.656009702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T12:40:40.657606645Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T12:40:52.571017852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:40832"
ts=2026-05-19T12:41:00.133789838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54792"
ts=2026-05-19T12:41:06.887212134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T12:41:36.890091465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:41:37.529851369Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:42:06.893908381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T12:42:25.230353369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:50300"
ts=2026-05-19T12:42:30.140096954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43778"
level=debug ts=2026-05-19T12:42:36.420793276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:42:36.897601783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T12:42:52.579741177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:38662"
ts=2026-05-19T12:42:55.233473225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:37536"
ts=2026-05-19T12:43:06.900935989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
ts=2026-05-19T12:43:30.145008155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52598"
level=debug ts=2026-05-19T12:43:34.712856732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T12:43:36.563045163Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T12:43:36.56451395Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T12:43:36.566273899Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T12:43:36.566296697Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T12:43:36.904481993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T12:43:55.23769559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:35472"
ts=2026-05-19T12:44:06.907180532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T12:44:25.832224866Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:44:36.910272414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:45:06.913456797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:45:08.408994539Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T12:45:08.410704735Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T12:45:30.152690899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49912"
ts=2026-05-19T12:45:36.916858654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:46:02.43555627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:46:06.920301281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T12:46:22.593232106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48918"
ts=2026-05-19T12:46:25.246683383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:58518"
ts=2026-05-19T12:46:30.157006584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60368"
ts=2026-05-19T12:46:36.923216078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:46:50.210245714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:47:06.926858089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T12:47:22.598817191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:57324"
ts=2026-05-19T12:47:25.251296133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:38078"
level=debug ts=2026-05-19T12:47:26.072053994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:47:36.929980591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:47:52.602393482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:42926"
ts=2026-05-19T12:48:00.163323456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:58578"
ts=2026-05-19T12:48:06.932677494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
level=debug ts=2026-05-19T12:48:10.029765397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:48:30.166182802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39078"
ts=2026-05-19T12:48:36.935307036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:48:55.25731163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:36420"
level=debug ts=2026-05-19T12:48:58.218765187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T12:48:58.220728381Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T12:49:00.169348467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:46482"
ts=2026-05-19T12:49:06.937969372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bdhh7-ad883517 10.129.0.67:7946"
ts=2026-05-19T12:49:25.260558668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:43622"
ts=2026-05-19T12:49:36.941518248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"
level=debug ts=2026-05-19T12:49:54.152125138Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T12:50:00.174261473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45734"
ts=2026-05-19T12:50:06.944841331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-675061f4 10.130.0.64:7946"
ts=2026-05-19T12:50:22.611402107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:58776"
ts=2026-05-19T12:50:25.264724854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:48298"
ts=2026-05-19T12:50:30.176930795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37578"
ts=2026-05-19T12:50:36.948056702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-8t2ss-e22a5ff0 10.130.0.63:7946"