level=info ts=2026-05-11T02:43:23.235104369Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-11T02:43:23.235156043Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-11T02:43:23.236945665Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-11T02:43:23.237264892Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-11T02:43:23.237373729Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.73 inf=eth0
level=debug ts=2026-05-11T02:43:23.23784871Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-11T02:43:23.237915403Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.73 inf=eth0
level=info ts=2026-05-11T02:43:23.237987715Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-46663528
ts=2026-05-11T02:43:23.238078375Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-11T02:43:23.238092915Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.73 advertisePort=7946
level=debug ts=2026-05-11T02:43:23.238113827Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.73 advertisePort=7946
level=info ts=2026-05-11T02:43:23.238280714Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-11T02:43:23.245776013Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-11T02:43:23.245798381Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-11T02:43:23.245813121Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=7.534929ms
level=error ts=2026-05-11T02:43:23.245825334Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-11T02:43:23.245842677Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-11T02:43:23.248724526Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T02:43:23.24873872Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-11T02:43:23.253297782Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-11T02:43:23.253302855Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-11T02:43:23.253317647Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-11T02:43:23.253339027Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-11T02:43:23.253346Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-11T02:43:23.253453873Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-11T02:43:23.2534665Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-05-11T02:43:23.253499674Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-11T02:43:23.253506721Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-05-11T02:43:23.253629479Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-11T02:43:23.253690373Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-11T02:43:23.253686358Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-11T02:43:23.2537072Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-11T02:43:23.253837981Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-11T02:43:23.25384704Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-11T02:43:23.253860431Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-11T02:43:24.254108558Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-11T02:43:24.357303175Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-11T02:43:24.357368538Z caller=loki.go:615 msg="Loki started" startup_time=1.193387879s
ts=2026-05-11T02:43:25.167904584Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T02:43:25.167926866Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T02:43:28.675697907Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T02:43:28.675719931Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-11T02:43:29.357579848Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-11T02:43:29.357639726Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
level=debug ts=2026-05-11T02:43:35.059133864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T02:43:36.211833712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.75:7946"
ts=2026-05-11T02:43:36.214276839Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.76:7946: dial tcp 10.128.0.76:7946: connect: connection refused"
level=info ts=2026-05-11T02:43:36.2142942Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=12.968444921s
ts=2026-05-11T02:43:46.173516317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:59828"
level=debug ts=2026-05-11T02:43:55.688684698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T02:44:02.606314069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41796"
level=debug ts=2026-05-11T02:44:19.448427935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T02:44:22.438672257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T02:44:42.779735818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T02:44:52.440653236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T02:45:06.288354699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:60282"
level=debug ts=2026-05-11T02:45:09.933755427Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-11T02:45:09.935548403Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T02:45:22.442445775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T02:45:32.612904492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40634"
ts=2026-05-11T02:45:52.44485295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T02:45:52.569049224Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T02:46:02.614463133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41274"
ts=2026-05-11T02:46:06.291988094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:57684"
ts=2026-05-11T02:46:22.446725347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T02:46:32.615608733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:56456"
ts=2026-05-11T02:46:36.294201272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:41920"
ts=2026-05-11T02:46:52.448157674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T02:46:59.439505893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T02:47:06.296139388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:41544"
ts=2026-05-11T02:47:22.450069347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T02:47:39.141017152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:38790"
ts=2026-05-11T02:47:52.45254002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T02:47:55.628274238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T02:48:02.622061083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:37410"
ts=2026-05-11T02:48:06.301136105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:47808"
ts=2026-05-11T02:48:22.455365879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T02:48:49.594356656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T02:48:52.457114341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T02:49:22.459121865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T02:49:52.100830798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-11T02:49:52.102361835Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T02:49:52.46155728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T02:50:22.464596171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T02:50:32.633037039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52340"
ts=2026-05-11T02:50:39.157685017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:38172"
level=debug ts=2026-05-11T02:50:50.516673515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T02:50:52.466219745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T02:51:06.317072485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:45998"
ts=2026-05-11T02:51:09.160286181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:47562"
ts=2026-05-11T02:51:22.468758601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T02:51:51.768319159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T02:51:52.471136231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T02:52:22.474045158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T02:52:36.32378527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:39122"
level=debug ts=2026-05-11T02:52:49.26573093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T02:52:52.476607024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T02:53:22.479052617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=info ts=2026-05-11T02:53:29.358363468Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-11T02:53:29.360522039Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T02:53:29.362291489Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T02:53:29.363981632Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T02:53:29.36400868Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-11T02:53:29.364021058Z caller=tables_manager.go:148 msg="compactor started"
level=debug ts=2026-05-11T02:53:31.421885296Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T02:53:32.646955096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:48142"
ts=2026-05-11T02:53:52.481821058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T02:54:02.648566159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53562"
ts=2026-05-11T02:54:22.484035143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T02:54:32.319545622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T02:54:32.321024819Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T02:54:52.485300871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T02:55:09.182858719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:39180"
level=debug ts=2026-05-11T02:55:19.736035376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T02:55:22.486808058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T02:55:39.185589456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:36564"
ts=2026-05-11T02:55:52.489356811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T02:56:02.656656051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43708"
level=debug ts=2026-05-11T02:56:05.257298429Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T02:56:06.34254532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:40648"
ts=2026-05-11T02:56:09.188081514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:43334"
ts=2026-05-11T02:56:22.491467679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T02:56:39.191058328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:49924"
ts=2026-05-11T02:56:52.493678052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T02:56:54.8883707Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T02:57:22.496129787Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T02:57:32.66422646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36324"
ts=2026-05-11T02:57:36.350628187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:42486"
ts=2026-05-11T02:57:39.197091058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:49254"
level=debug ts=2026-05-11T02:57:47.618840311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T02:57:52.498192843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T02:58:06.35301731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:43174"
ts=2026-05-11T02:58:22.500307168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T02:58:34.225357959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T02:58:34.227220528Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T02:58:52.502107908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T02:59:09.206197013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:58884"
ts=2026-05-11T02:59:22.50416806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T02:59:30.945094291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T02:59:36.360058746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:59680"
ts=2026-05-11T02:59:52.506519344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:00:02.675812491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58910"
level=debug ts=2026-05-11T03:00:16.080416231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:00:22.50918629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:00:39.214818096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:34260"
ts=2026-05-11T03:00:52.511178313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:01:02.680431254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46006"
level=debug ts=2026-05-11T03:01:04.580267411Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:01:22.513157975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:01:32.682007563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:54598"
ts=2026-05-11T03:01:39.220560721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:35878"
ts=2026-05-11T03:01:52.515614341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T03:02:00.678119077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:02:06.374848369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:41692"
ts=2026-05-11T03:02:22.518814993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:02:36.376608459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:35026"
level=debug ts=2026-05-11T03:02:40.853958821Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T03:02:40.855539815Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:02:52.521262997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:03:06.378677889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:34724"
level=debug ts=2026-05-11T03:03:17.263537369Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:03:22.523824857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=error ts=2026-05-11T03:03:29.367955602Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:03:29.37011791Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:03:29.371943328Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:03:29.371967453Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:03:52.525363389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:04:02.694273721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58070"
level=debug ts=2026-05-11T03:04:03.863880343Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:04:22.526710137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:04:32.69618239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39722"
ts=2026-05-11T03:04:52.52844381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T03:05:05.667134561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:05:22.530868508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:05:36.39491762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:37004"
ts=2026-05-11T03:05:52.534162386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T03:05:53.413032551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:06:02.703575201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:37790"
ts=2026-05-11T03:06:22.536910202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T03:06:28.387416275Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T03:06:28.389452507Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:06:36.400442928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:37798"
ts=2026-05-11T03:06:39.253023774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:54146"
ts=2026-05-11T03:06:52.538403839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:07:06.402217929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:49238"
level=debug ts=2026-05-11T03:07:19.35940156Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:07:22.540917832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:07:36.404669562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:45180"
ts=2026-05-11T03:07:52.543153491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:08:09.262805252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:34988"
level=debug ts=2026-05-11T03:08:17.1121154Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:08:22.544704352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:08:36.409065493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:55866"
ts=2026-05-11T03:08:52.546891036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:09:02.719470273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:56838"
ts=2026-05-11T03:09:09.268701972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:35118"
level=debug ts=2026-05-11T03:09:12.984741195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:09:22.549557012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:09:39.272111096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:58766"
ts=2026-05-11T03:09:52.552481253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T03:09:52.672549088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:10:02.724171857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52054"
ts=2026-05-11T03:10:22.554753271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:10:32.72591084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39334"
level=debug ts=2026-05-11T03:10:39.745684075Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T03:10:39.747387486Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:10:52.55757886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:11:06.422956798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:49474"
ts=2026-05-11T03:11:09.282492104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:45486"
ts=2026-05-11T03:11:22.560870362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T03:11:22.648769282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:11:32.729358309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41880"
ts=2026-05-11T03:11:39.285632283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:44090"
ts=2026-05-11T03:11:52.562490138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T03:12:10.311070801Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:12:22.56419989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:12:52.56652634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T03:12:59.664046419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:13:06.434008078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:32796"
ts=2026-05-11T03:13:22.568554384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=error ts=2026-05-11T03:13:29.369285661Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:13:29.371191622Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:13:29.372877086Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:13:29.372900147Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:13:32.738157598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:50498"
level=debug ts=2026-05-11T03:13:37.444640353Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:13:39.310143071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:58916"
ts=2026-05-11T03:13:52.570797142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:14:09.313573231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:57658"
ts=2026-05-11T03:14:22.572688818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:14:32.741942816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59184"
level=debug ts=2026-05-11T03:14:34.749893854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T03:14:34.751706761Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:14:52.574939632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:15:02.744005092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42564"
ts=2026-05-11T03:15:09.31881692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:48568"
ts=2026-05-11T03:15:22.57713697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T03:15:35.921237131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:15:52.579303743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:16:22.581589498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T03:16:25.273217905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:16:39.327977778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:51758"
ts=2026-05-11T03:16:52.583717657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:17:06.454156936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:41960"
ts=2026-05-11T03:17:09.331408559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:44638"
level=debug ts=2026-05-11T03:17:17.688294919Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:17:22.586408763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:17:52.588740508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:18:02.75877687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:55600"
level=debug ts=2026-05-11T03:18:06.427503144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:18:06.457695423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:39380"
ts=2026-05-11T03:18:09.340470492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:32892"
ts=2026-05-11T03:18:22.591872356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T03:18:45.874366073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T03:18:45.876569376Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:18:52.594693422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:19:02.763543313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:48854"
ts=2026-05-11T03:19:09.347244908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:54742"
ts=2026-05-11T03:19:22.597149183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T03:19:32.742841065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:19:39.350368773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:33598"
ts=2026-05-11T03:19:52.59883539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:20:02.767585904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40676"
ts=2026-05-11T03:20:06.467356988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:42386"
ts=2026-05-11T03:20:22.60105085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T03:20:22.654376109Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:20:52.603094957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:21:02.772471021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:37774"
level=debug ts=2026-05-11T03:21:21.401509412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:21:22.605838925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:21:39.361357578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:36510"
ts=2026-05-11T03:21:52.608314894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T03:22:06.053068348Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:22:06.478527042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:42412"
ts=2026-05-11T03:22:22.609810729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:22:52.611696673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T03:23:01.376477727Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T03:23:01.378596851Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:23:22.613570609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=error ts=2026-05-11T03:23:29.368696993Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:23:29.371114486Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:23:29.373349814Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:23:29.373374207Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:23:32.783823049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:50352"
level=debug ts=2026-05-11T03:23:37.829985615Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:23:52.616120523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:24:22.617894456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T03:24:30.311683581Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:24:39.377716393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:55756"
ts=2026-05-11T03:24:52.619136256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:25:22.619963542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T03:25:24.026025959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:25:52.622240357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:26:02.795073923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:56422"
ts=2026-05-11T03:26:06.502722428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:51544"
ts=2026-05-11T03:26:09.385210784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:44470"
ts=2026-05-11T03:26:22.624980674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T03:26:25.188646413Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:26:32.796125594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45512"
ts=2026-05-11T03:26:39.388079531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:45418"
ts=2026-05-11T03:26:52.6276848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:27:02.79760597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49200"
ts=2026-05-11T03:27:22.630199427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T03:27:26.314200145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T03:27:26.316097663Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:27:52.633779997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T03:28:09.282415779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:28:22.636229275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:28:39.405476871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:42500"
level=debug ts=2026-05-11T03:28:48.703105152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:28:52.638003435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:29:02.8076105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58068"
ts=2026-05-11T03:29:22.640658551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:29:36.520585055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:40252"
level=debug ts=2026-05-11T03:29:51.171935022Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:29:52.643168694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:30:06.522561705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:53060"
ts=2026-05-11T03:30:22.645041122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T03:30:25.753018302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:30:32.813657924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52296"
ts=2026-05-11T03:30:36.524847151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:51630"
ts=2026-05-11T03:30:52.647113935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:31:09.419983628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:48524"
level=debug ts=2026-05-11T03:31:21.719719141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T03:31:21.72118902Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:31:22.64877287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:31:36.529153241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:34540"
ts=2026-05-11T03:31:39.423443052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:37818"
ts=2026-05-11T03:31:52.651006529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T03:32:03.403415058Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:32:06.531087283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:33628"
ts=2026-05-11T03:32:22.653928513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:32:52.655743892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T03:33:05.166544634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:33:22.658219883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=error ts=2026-05-11T03:33:29.368652794Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:33:29.370783048Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:33:29.372856821Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:33:29.372877296Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:33:39.434927951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:35282"
level=debug ts=2026-05-11T03:33:50.826290909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:33:52.66082955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:34:06.538916969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:42860"
ts=2026-05-11T03:34:22.663042237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:34:36.542310365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:55864"
ts=2026-05-11T03:34:52.665180525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T03:34:52.70967055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:35:02.836847578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43428"
ts=2026-05-11T03:35:22.666584553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T03:35:33.727649201Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T03:35:33.729270965Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:35:52.667654531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:36:02.840629944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59928"
level=debug ts=2026-05-11T03:36:15.563613753Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:36:22.668782603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:36:39.451422068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:55760"
ts=2026-05-11T03:36:52.670251062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T03:36:57.409553552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:37:09.454014504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:56618"
ts=2026-05-11T03:37:22.672155972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T03:37:34.447020812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:37:52.674269488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T03:38:20.059629496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:38:22.676271733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:38:32.85155256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47472"
ts=2026-05-11T03:38:36.566529106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:44908"
ts=2026-05-11T03:38:39.461815061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:60242"
ts=2026-05-11T03:38:52.677794441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:39:02.853239726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34130"
level=debug ts=2026-05-11T03:39:08.386855627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T03:39:08.388382486Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:39:09.464831982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:39742"
ts=2026-05-11T03:39:22.678819673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:39:32.854371168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45552"
ts=2026-05-11T03:39:52.681442866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:40:02.855993483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:38072"
level=debug ts=2026-05-11T03:40:10.671472907Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:40:22.683178687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:40:36.579163083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:57780"
ts=2026-05-11T03:40:39.473310711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:50530"
ts=2026-05-11T03:40:52.684265718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T03:40:57.825934981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:41:09.476632718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:59440"
ts=2026-05-11T03:41:22.686095151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:41:36.583434569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:54026"
level=debug ts=2026-05-11T03:41:47.983679326Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:41:52.687629991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:42:02.865301539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:56730"
ts=2026-05-11T03:42:22.689482729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T03:42:38.90340887Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:42:39.485509648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:60504"
ts=2026-05-11T03:42:52.691783055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:43:06.592513866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:58680"
level=debug ts=2026-05-11T03:43:15.952190166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T03:43:15.954075409Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:43:22.694060771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=error ts=2026-05-11T03:43:29.368342961Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:43:29.370382089Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:43:29.372085343Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:43:29.372108675Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:43:52.696401639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T03:43:54.337833141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:44:22.698481585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T03:44:32.283386944Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:44:52.700790309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:45:02.880077412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41910"
ts=2026-05-11T03:45:09.499037419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:40112"
level=debug ts=2026-05-11T03:45:09.64295501Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:45:22.701731249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:45:32.88133938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47426"
ts=2026-05-11T03:45:39.501525813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:39280"
ts=2026-05-11T03:45:52.703564321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:46:02.883033811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:57546"
level=debug ts=2026-05-11T03:46:06.370069821Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:46:22.705780192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:46:39.507000943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:57664"
ts=2026-05-11T03:46:52.707222531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T03:46:56.515354206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T03:46:56.517153757Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:47:02.887636055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34200"
ts=2026-05-11T03:47:22.70955178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:47:32.888554595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45632"
ts=2026-05-11T03:47:39.512501938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:40002"
level=debug ts=2026-05-11T03:47:40.340321373Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:47:52.712136707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T03:48:19.366069815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:48:22.714269018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:48:36.621923208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:53590"
ts=2026-05-11T03:48:52.716413834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:49:02.894077662Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43346"
ts=2026-05-11T03:49:06.624454293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:46458"
level=debug ts=2026-05-11T03:49:16.38245077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:49:22.718803862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:49:52.72043152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T03:50:03.589837975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:50:06.631110169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:37536"
ts=2026-05-11T03:50:22.722352351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:50:36.632773318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:44456"
ts=2026-05-11T03:50:39.532019887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:33348"
level=debug ts=2026-05-11T03:50:51.928954632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T03:50:51.93087173Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:50:52.724608932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:51:02.90373785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45656"
ts=2026-05-11T03:51:06.635467477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:42184"
ts=2026-05-11T03:51:22.727564881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T03:51:52.242857563Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:51:52.729071597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:52:06.640264175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:44474"
ts=2026-05-11T03:52:22.730674512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T03:52:46.025960394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:52:52.732646216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:53:06.644203462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:56902"
ts=2026-05-11T03:53:22.7341264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=error ts=2026-05-11T03:53:29.368300497Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:53:29.370321313Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:53:29.372084748Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T03:53:29.372108784Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:53:32.915970936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:60738"
ts=2026-05-11T03:53:36.646344441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:35542"
level=debug ts=2026-05-11T03:53:37.338769388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:53:52.736708434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:54:02.917216748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59460"
ts=2026-05-11T03:54:06.647935691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:60042"
level=debug ts=2026-05-11T03:54:16.173222199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:54:22.739640498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T03:54:32.918793998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51536"
ts=2026-05-11T03:54:52.742530301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:55:02.919618061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39646"
level=debug ts=2026-05-11T03:55:14.04610544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T03:55:14.047888941Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:55:22.743727847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:55:32.920855153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36652"
ts=2026-05-11T03:55:36.657483889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:37252"
level=debug ts=2026-05-11T03:55:51.399125032Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:55:52.74485884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:56:09.564636486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:41660"
ts=2026-05-11T03:56:22.74667111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T03:56:29.709258244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:56:32.924505901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:35956"
ts=2026-05-11T03:56:36.662089433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:54188"
ts=2026-05-11T03:56:39.567583269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:40988"
ts=2026-05-11T03:56:52.748486132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:57:06.664491595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:47254"
ts=2026-05-11T03:57:09.570846661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:52382"
level=debug ts=2026-05-11T03:57:11.151714052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:57:22.750205605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T03:57:50.917910783Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:57:52.75223526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:58:02.931256642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36748"
ts=2026-05-11T03:58:22.753657641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T03:58:32.933427215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59290"
level=debug ts=2026-05-11T03:58:33.489378467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T03:58:33.493020982Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T03:58:52.755916698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T03:59:02.935004879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36262"
ts=2026-05-11T03:59:22.758283469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T03:59:23.618873857Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T03:59:52.760773776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T04:00:02.939649733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:38974"
ts=2026-05-11T04:00:09.588268454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:36218"
level=debug ts=2026-05-11T04:00:22.383458464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:00:22.763695848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T04:00:36.683402215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:36458"
ts=2026-05-11T04:00:52.76584397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T04:01:02.943461703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:37392"
level=debug ts=2026-05-11T04:01:03.304144681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:01:22.768299953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T04:01:39.596019527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:45594"
level=debug ts=2026-05-11T04:01:45.082203398Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:01:52.771077362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T04:02:22.774193393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T04:02:32.951666318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47718"
ts=2026-05-11T04:02:36.703571795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:45598"
level=debug ts=2026-05-11T04:02:41.510262487Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T04:02:41.511707957Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T04:02:52.776094048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T04:03:02.953366007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:55550"
ts=2026-05-11T04:03:22.778264906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=error ts=2026-05-11T04:03:29.368088851Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T04:03:29.370070903Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T04:03:29.372471797Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T04:03:29.372492826Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-11T04:03:34.224899952Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:03:36.707298923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:53390"
ts=2026-05-11T04:03:52.780748225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T04:04:09.610213383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:34812"
level=debug ts=2026-05-11T04:04:14.237619061Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:04:22.782025369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T04:04:32.960723997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33064"
ts=2026-05-11T04:04:52.783879278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T04:04:54.487927208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:05:09.616865088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:43502"
ts=2026-05-11T04:05:22.786171095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T04:05:29.417538067Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:05:36.718800282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:50372"
ts=2026-05-11T04:05:39.619791877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:39074"
ts=2026-05-11T04:05:52.788737106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T04:06:06.723969534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:45948"
ts=2026-05-11T04:06:22.791157908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T04:06:29.357397257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T04:06:29.358847815Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T04:06:39.62544848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:34456"
ts=2026-05-11T04:06:52.793798086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T04:07:22.797511473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T04:07:28.629995622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:07:32.975554869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:35338"
ts=2026-05-11T04:07:52.799365921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T04:08:02.976590892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51966"
level=debug ts=2026-05-11T04:08:05.17187617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:08:22.800973092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T04:08:52.802516141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T04:08:54.759350722Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:09:02.979941611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58280"
ts=2026-05-11T04:09:22.804751708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T04:09:40.658068038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:09:52.806640332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T04:10:09.644826606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:37266"
ts=2026-05-11T04:10:22.80980568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T04:10:37.731605389Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T04:10:37.733286797Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T04:10:52.812364984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T04:11:14.384380145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:11:22.815108772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T04:11:32.990850752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:60172"
ts=2026-05-11T04:11:52.81739996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T04:11:53.493470833Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:12:09.65552446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:43560"
ts=2026-05-11T04:12:22.820655546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T04:12:30.143141003Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:12:32.994762629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:48298"
ts=2026-05-11T04:12:36.767805476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:40586"
ts=2026-05-11T04:12:52.823381382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T04:13:22.826173681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T04:13:27.166426918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T04:13:29.36815766Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T04:13:29.36973513Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T04:13:29.371264051Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T04:13:29.371285287Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T04:13:52.828128786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
level=debug ts=2026-05-11T04:14:07.032687109Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T04:14:07.03415619Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T04:14:09.665789009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:38420"
ts=2026-05-11T04:14:22.829515063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T04:14:39.668105462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:49870"
ts=2026-05-11T04:14:52.831430339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T04:15:03.008575319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34350"
level=debug ts=2026-05-11T04:15:04.826003394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:15:06.780065181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:42264"
ts=2026-05-11T04:15:22.834465608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T04:15:36.782345202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:44712"
level=debug ts=2026-05-11T04:15:52.015203844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:15:52.836912478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T04:16:06.785638236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:50242"
ts=2026-05-11T04:16:22.838531638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T04:16:30.597428108Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:16:52.840375782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T04:17:03.018688545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42724"
ts=2026-05-11T04:17:06.794756685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:42096"
level=debug ts=2026-05-11T04:17:12.656196904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:17:22.84349721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T04:17:36.797495181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:34026"
ts=2026-05-11T04:17:52.845988835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T04:18:09.6928794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:55806"
level=debug ts=2026-05-11T04:18:10.701468849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T04:18:10.703192691Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T04:18:22.848524737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T04:18:33.025053997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:57546"
ts=2026-05-11T04:18:36.802712962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:35292"
ts=2026-05-11T04:18:52.851727496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T04:18:59.062581759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:19:22.854532183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T04:19:46.595059033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:19:52.856245962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T04:20:09.704194076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:58138"
ts=2026-05-11T04:20:22.857951599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
level=debug ts=2026-05-11T04:20:25.240754406Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:20:33.034485947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33610"
ts=2026-05-11T04:20:52.859782499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T04:21:03.036227406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:55910"
level=debug ts=2026-05-11T04:21:09.499084274Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:21:22.86192929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T04:21:36.826666032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:47078"
ts=2026-05-11T04:21:39.712786276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:37688"
ts=2026-05-11T04:21:52.865038652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T04:22:03.041060613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45596"
level=debug ts=2026-05-11T04:22:03.617036277Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T04:22:03.618914288Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T04:22:06.831140999Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:34484"
ts=2026-05-11T04:22:22.866850255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T04:22:33.042611098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:50952"
level=debug ts=2026-05-11T04:22:52.366415002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:22:52.86810566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
ts=2026-05-11T04:23:03.044034379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36678"
ts=2026-05-11T04:23:22.869272837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"
level=debug ts=2026-05-11T04:23:27.439957903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T04:23:29.367228103Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T04:23:29.369124523Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T04:23:29.370883555Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T04:23:29.370903995Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T04:23:52.871435394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-mbhcr-cd657c25 10.128.0.75:7946"
ts=2026-05-11T04:24:03.048358307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51406"
level=debug ts=2026-05-11T04:24:08.641548998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:24:22.874238701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T04:24:33.050263241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:60460"
ts=2026-05-11T04:24:39.730629431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:46284"
ts=2026-05-11T04:24:52.876024583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-50286aab 10.129.0.70:7946"
ts=2026-05-11T04:25:03.051591489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39964"
ts=2026-05-11T04:25:06.852255256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:43744"
level=debug ts=2026-05-11T04:25:07.71402056Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T04:25:22.878840363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-95w97-467fa0d2 10.130.0.69:7946"